{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-03-22T07:25:21.947883Z",
     "start_time": "2025-03-22T07:25:17.799813Z"
    }
   },
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "\n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)\n",
    "np.random.seed(seed)\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=12, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.0\n",
      "torch 2.3.1+cu121\n",
      "cuda:0\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "## 数据加载\n",
    "\n",
    "- 采用WMT16的德语和英语平行语料库，数据集主页：[WMT16](https://www.statmt.org/wmt16/multimodal-task.html#task1)"
   ],
   "id": "e6a32f2a25b33149"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-21T01:17:12.228052Z",
     "start_time": "2025-03-21T01:16:52.997470Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 运行脚本对wtm16数据集进行分词\n",
    "!sh data_multi30k.sh wmt16 wmt16_cut de en"
   ],
   "id": "7d930e0449ecaeb2",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[train] 源语言文本分词完成\n",
      "[train] 目标语言文本分词完成\n",
      "[val] 源语言文本分词完成\n",
      "[val] 目标语言文本分词完成\n",
      "[test] 源语言文本分词完成\n",
      "[test] 目标语言文本分词完成\n",
      "Finished applying bpe to train files.\n",
      "Finished applying bpe to val files.\n",
      "Finished applying bpe to test files.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/20000 [00:00<?, ?it/s]\n",
      "  0%|          | 21/20000 [00:00<01:40, 198.67it/s]\n",
      "  0%|          | 78/20000 [00:00<00:48, 410.84it/s]\n",
      "  1%|          | 181/20000 [00:00<00:28, 689.20it/s]\n",
      "  2%|1         | 319/20000 [00:00<00:20, 958.63it/s]\n",
      "  2%|2         | 460/20000 [00:00<00:17, 1117.93it/s]\n",
      "  3%|3         | 625/20000 [00:00<00:14, 1293.57it/s]\n",
      "  4%|4         | 804/20000 [00:00<00:13, 1452.52it/s]\n",
      "  5%|5         | 1019/20000 [00:00<00:11, 1673.01it/s]\n",
      "  6%|6         | 1263/20000 [00:00<00:09, 1908.52it/s]\n",
      "  8%|7         | 1501/20000 [00:01<00:09, 2047.24it/s]\n",
      "  9%|9         | 1841/20000 [00:01<00:07, 2458.35it/s]\n",
      " 11%|#         | 2181/20000 [00:01<00:06, 2739.33it/s]\n",
      " 13%|#3        | 2627/20000 [00:01<00:05, 3257.39it/s]\n",
      " 16%|#6        | 3262/20000 [00:01<00:03, 4186.85it/s]\n",
      " 19%|#8        | 3761/20000 [00:01<00:03, 4315.96it/s]\n",
      " 21%|##        | 4193/20000 [00:01<00:05, 3108.72it/s]\n",
      " 23%|##2       | 4551/20000 [00:01<00:05, 2719.77it/s]\n",
      " 24%|##4       | 4862/20000 [00:02<00:06, 2518.92it/s]\n",
      " 26%|##5       | 5141/20000 [00:02<00:06, 2448.13it/s]\n",
      " 27%|##7       | 5404/20000 [00:02<00:06, 2418.06it/s]\n",
      " 28%|##8       | 5664/20000 [00:02<00:05, 2458.84it/s]\n",
      " 30%|##9       | 5935/20000 [00:02<00:05, 2519.04it/s]\n",
      " 31%|###1      | 6241/20000 [00:02<00:05, 2658.94it/s]\n",
      " 33%|###2      | 6568/20000 [00:02<00:04, 2819.97it/s]\n",
      " 35%|###4      | 6938/20000 [00:02<00:04, 3063.68it/s]\n",
      " 37%|###6      | 7363/20000 [00:02<00:03, 3401.49it/s]\n",
      " 40%|###9      | 7978/20000 [00:03<00:02, 4185.95it/s]\n",
      " 42%|####2     | 8403/20000 [00:03<00:03, 3623.86it/s]\n",
      " 44%|####3     | 8783/20000 [00:03<00:03, 2941.03it/s]\n",
      " 46%|####5     | 9108/20000 [00:03<00:04, 2654.92it/s]\n",
      " 47%|####6     | 9397/20000 [00:03<00:04, 2556.16it/s]\n",
      " 48%|####8     | 9668/20000 [00:03<00:04, 2538.47it/s]\n",
      " 50%|####9     | 9933/20000 [00:03<00:03, 2542.22it/s]\n",
      " 51%|#####1    | 10218/20000 [00:03<00:03, 2617.53it/s]\n",
      " 53%|#####2    | 10530/20000 [00:04<00:03, 2746.67it/s]\n",
      " 54%|#####4    | 10872/20000 [00:04<00:03, 2932.97it/s]\n",
      " 56%|#####6    | 11292/20000 [00:04<00:02, 3278.93it/s]\n",
      " 59%|#####8    | 11784/20000 [00:04<00:02, 3750.45it/s]\n",
      " 63%|######2   | 12595/20000 [00:04<00:01, 5008.19it/s]\n",
      " 66%|######5   | 13103/20000 [00:04<00:02, 2827.78it/s]\n",
      " 68%|######7   | 13501/20000 [00:05<00:02, 2554.64it/s]\n",
      " 69%|######9   | 13839/20000 [00:05<00:02, 2467.97it/s]\n",
      " 71%|#######   | 14142/20000 [00:05<00:02, 2447.80it/s]\n",
      " 72%|#######2  | 14426/20000 [00:05<00:02, 2480.39it/s]\n",
      " 74%|#######3  | 14703/20000 [00:05<00:02, 2512.71it/s]\n",
      " 75%|#######4  | 14990/20000 [00:05<00:01, 2598.15it/s]\n",
      " 77%|#######6  | 15305/20000 [00:05<00:01, 2735.77it/s]\n",
      " 78%|#######8  | 15651/20000 [00:05<00:01, 2929.85it/s]\n",
      " 80%|########  | 16001/20000 [00:05<00:01, 3081.39it/s]\n",
      " 82%|########2 | 16497/20000 [00:06<00:00, 3606.62it/s]\n",
      " 86%|########6 | 17210/20000 [00:06<00:00, 3730.49it/s]\n",
      " 88%|########7 | 17587/20000 [00:06<00:00, 2540.26it/s]\n",
      " 89%|########9 | 17890/20000 [00:06<00:00, 2146.68it/s]\n",
      " 91%|######### | 18144/20000 [00:06<00:00, 1944.02it/s]\n",
      " 92%|#########1| 18365/20000 [00:07<00:00, 1848.26it/s]\n",
      " 93%|#########2| 18567/20000 [00:07<00:00, 1803.34it/s]\n",
      " 94%|#########3| 18758/20000 [00:07<00:00, 1773.49it/s]\n",
      " 95%|#########4| 18942/20000 [00:07<00:00, 1767.56it/s]\n",
      " 96%|#########5| 19124/20000 [00:07<00:00, 1777.37it/s]\n",
      " 97%|#########6| 19315/20000 [00:07<00:00, 1809.66it/s]\n",
      " 98%|#########7| 19513/20000 [00:07<00:00, 1852.74it/s]\n",
      " 99%|#########8| 19724/20000 [00:07<00:00, 1920.68it/s]\n",
      "100%|#########9| 19946/20000 [00:07<00:00, 2003.98it/s]\n",
      "100%|##########| 20000/20000 [00:07<00:00, 2527.38it/s]\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## Dataset",
   "id": "c30971f51552bb90"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:39.465751Z",
     "start_time": "2025-03-24T10:58:39.447432Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from pathlib import Path\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "\n",
    "class LangPairDataset(Dataset):\n",
    "\n",
    "    def __init__(\n",
    "            self, mode=\"train\", max_length=128, overwrite_cache=False, data_dir=\"wmt16\",\n",
    "    ):\n",
    "        self.data_dir = Path(data_dir)\n",
    "        cache_path = self.data_dir / \".cache\" / f\"de2en_{mode}_{max_length}.npy\"\n",
    "\n",
    "        if overwrite_cache or not cache_path.exists():\n",
    "            cache_path.parent.mkdir(parents=True, exist_ok=True)  # 创建缓存目录\n",
    "\n",
    "            with open(self.data_dir / f\"{mode}_src.bpe\", \"r\", encoding=\"utf8\") as file:\n",
    "                self.src = file.readlines()  # 读取源语言文件所有行\n",
    "\n",
    "            with open(self.data_dir / f\"{mode}_trg.bpe\", \"r\", encoding=\"utf8\") as file:\n",
    "                self.trg = file.readlines()  # 读取目标语言文件所有行\n",
    "\n",
    "            filtered_src = []\n",
    "            filtered_trg = []\n",
    "            # max length filter,超出最大长度的句子舍弃\n",
    "            # zip() 函数用于将可迭代的对象作为参数，将对象中对应的元素打包成一个个元组，然后返回由这些元组组成的列表。\n",
    "            #[src, trg] = zip(self.src, self.trg)\n",
    "            for src, trg in zip(self.src, self.trg):\n",
    "                if len(src) <= max_length and len(trg) <= max_length:  # 过滤长度超过最大长度的句子\n",
    "                    filtered_src.append(src.strip())  # 去掉句子前后的空格\n",
    "                    filtered_trg.append(trg.strip())\n",
    "            filtered_src = np.array(filtered_src)\n",
    "            filtered_trg = np.array(filtered_trg)\n",
    "            np.save(\n",
    "                cache_path,\n",
    "                {\"src\": filtered_src, \"trg\": filtered_trg},\n",
    "                # allo_pickle=True允许保存对象数组，将过滤后的数据保存为 NumPy 数组，存储在缓存文件中\n",
    "                allow_pickle=True,\n",
    "            )\n",
    "            print(f\"save cache to {cache_path}\")\n",
    "\n",
    "        else:\n",
    "            #allow_pickle=True允许保存对象数组\n",
    "            cache_dict = np.load(cache_path, allow_pickle=True).item()\n",
    "            print(f\"load {mode} dataset from {cache_path}\")\n",
    "            filtered_src = cache_dict[\"src\"]\n",
    "            filtered_trg = cache_dict[\"trg\"]\n",
    "\n",
    "        self.src = filtered_src\n",
    "        self.trg = filtered_trg\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        return self.src[index], self.trg[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.src)\n",
    "\n",
    "\n",
    "train_ds = LangPairDataset(\"train\")\n",
    "val_ds = LangPairDataset(\"val\")"
   ],
   "id": "a609183b670d5200",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load train dataset from wmt16\\.cache\\de2en_train_128.npy\n",
      "load val dataset from wmt16\\.cache\\de2en_val_128.npy\n"
     ]
    }
   ],
   "execution_count": 350
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "# Tokenizer\n",
    "\n",
    "这里有两种处理方式，分别对应着 encoder 和 decoder 的 word embedding 是否共享，这里实现共享的方案"
   ],
   "id": "6f04df25591f10b0"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:39.688556Z",
     "start_time": "2025-03-24T10:58:39.672582Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#载入词表，看下词表长度，词表就像英语字典,构建word2idx和idx2word\n",
    "word2idx = {\n",
    "    \"[PAD]\": 0,  # 填充 token\n",
    "    \"[BOS]\": 1,  # begin of sentence\n",
    "    \"[UNK]\": 2,  # 未知 token\n",
    "    \"[EOS]\": 3,  # end of sentence\n",
    "}\n",
    "idx2word = {value: key for key, value in word2idx.items()}\n",
    "index = len(idx2word)\n",
    "threshold = 1  # 出现次数低于此的token舍弃\n",
    "\n",
    "# 构建一个经过筛选的词汇表，并为每个词汇分配一个唯一的索引，便于后续的自然语言处理任务（如机器翻译）中使用。\n",
    "# 打开名为 \"wmt16/vocab\" 的文件，使用 UTF-8 编码读取\n",
    "with open(\"wmt16/vocab\", \"r\", encoding=\"utf8\") as file:\n",
    "    # 使用 tqdm 进度条逐行读取文件内容\n",
    "    for line in tqdm(file.readlines()):\n",
    "        # 去除每行的首尾空白字符，并按空格分割成两部分：token 和 counts\n",
    "        token, counts = line.strip().split()\n",
    "        # 如果 counts 转换为整数后大于等于设定的阈值 threshold\n",
    "        if int(counts) >= threshold:\n",
    "            # 将 token 映射到当前的 index，并存储在 word2idx 字典中\n",
    "            word2idx[token] = index\n",
    "            # 将当前的 index 映射到 token，并存储在 idx2word 字典中\n",
    "            idx2word[index] = token\n",
    "            # index 自增 1，为下一个 token 准备\n",
    "            index += 1\n",
    "\n",
    "# 计算 word2idx 字典的长度，即词汇表的大小\n",
    "vocab_size = len(word2idx)\n",
    "# 打印词汇表的大小\n",
    "print(\"vocab_size: {}\".format(vocab_size))\n"
   ],
   "id": "11588c661242705",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18107/18107 [00:00<00:00, 2012141.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_size: 18111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "execution_count": 351
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:39.694912Z",
     "start_time": "2025-03-24T10:58:39.689560Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class Tokenizer:\n",
    "    def __init__(self, word2idx, idx2word, max_length=128, pad_idx=0, bos_idx=1, eos_idx=3, unk_idx=2):\n",
    "        self.word2idx = word2idx\n",
    "        self.idx2word = idx2word\n",
    "        self.max_length = max_length\n",
    "        self.pad_idx = pad_idx\n",
    "        self.bos_idx = bos_idx\n",
    "        self.eos_idx = eos_idx\n",
    "        self.unk_idx = unk_idx\n",
    "\n",
    "    def encode(self, text_list, padding_first=False, add_bos=True, add_eos=True, return_mask=False):\n",
    "        \"\"\"如果padding_first == True，则padding加载前面，否则加载后面\"\"\"\n",
    "        max_length = min(self.max_length, add_eos + add_bos + max([len(text) for text in text_list]))\n",
    "        indices_list = []\n",
    "        for text in text_list:\n",
    "            indices = [self.word2idx.get(word, self.unk_idx) for word in text[:max_length - add_bos - add_eos]]\n",
    "            if add_bos:\n",
    "                indices = [self.bos_idx] + indices\n",
    "            if add_eos:\n",
    "                indices = indices + [self.eos_idx]\n",
    "            if padding_first:\n",
    "                indices = [self.pad_idx] * (max_length - len(indices)) + indices\n",
    "            else:\n",
    "                indices = indices + [self.pad_idx] * (max_length - len(indices))\n",
    "            indices_list.append(indices)\n",
    "        input_ids = torch.tensor(indices_list)\n",
    "        # masks \n",
    "        masks = (input_ids == self.pad_idx).to(dtype=torch.int64)  # 为了方便损失计算，这里的mask为0的地方需要计算，为1的地方不需要计算\n",
    "        return input_ids if not return_mask else (input_ids, masks)\n",
    "\n",
    "    def decode(self, indices_list, remove_bos=True, remove_eos=True, remove_pad=True, split=False):\n",
    "        text_list = []\n",
    "        for indices in indices_list:\n",
    "            text = []\n",
    "            for index in indices:\n",
    "                word = self.idx2word.get(index, \"[UNK]\")\n",
    "                if remove_bos and word == \"[BOS]\":\n",
    "                    continue\n",
    "                if remove_eos and word == \"[EOS]\":\n",
    "                    break\n",
    "                if remove_pad and word == \"[PAD]\":\n",
    "                    break\n",
    "                text.append(word)\n",
    "            text_list.append(\" \".join(text) if not split else text)\n",
    "        return text_list"
   ],
   "id": "42f8d1991115387d",
   "outputs": [],
   "execution_count": 352
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:39.712948Z",
     "start_time": "2025-03-24T10:58:39.707931Z"
    }
   },
   "cell_type": "code",
   "source": [
    "tokenizer = Tokenizer(word2idx=word2idx, idx2word=idx2word)\n",
    "\n",
    "raw_text = [\"hello world\".split(), \"tokenize text datas with batch\".split(), \"this is a test\".split()]\n",
    "indices = tokenizer.encode(raw_text, padding_first=False, add_bos=True, add_eos=True)\n",
    "decode_text = tokenizer.decode(indices.tolist(), remove_bos=False, remove_eos=False, remove_pad=False)\n",
    "print(\"raw text\")\n",
    "for raw in raw_text:\n",
    "    print(raw)\n",
    "print(\"indices\")\n",
    "for index in indices:\n",
    "    print(index)\n",
    "print(\"decode text\")\n",
    "for decode in decode_text:\n",
    "    print(decode)"
   ],
   "id": "ac20ab81aae37728",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "raw text\n",
      "['hello', 'world']\n",
      "['tokenize', 'text', 'datas', 'with', 'batch']\n",
      "['this', 'is', 'a', 'test']\n",
      "indices\n",
      "tensor([   1, 9458, 3522,    3,    0,    0,    0])\n",
      "tensor([   1,    2, 5463,    2,   22,    2,    3])\n",
      "tensor([   1,  385,   18,    5, 5699,    3,    0])\n",
      "decode text\n",
      "[BOS] hello world [EOS] [PAD] [PAD] [PAD]\n",
      "[BOS] [UNK] text [UNK] with [UNK] [EOS]\n",
      "[BOS] this is a test [EOS] [PAD]\n"
     ]
    }
   ],
   "execution_count": 353
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "# Transformer Batch Sampler\n",
    "\n",
    "句子按照序列长度差不多的分到一个批次。 每个训练批次包含一组句子对，其中包含大约 25000 个源标记和 25000 个目标标记"
   ],
   "id": "a2d074be54751af0"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:39.839641Z",
     "start_time": "2025-03-24T10:58:39.835138Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class SampleInfo:\n",
    "    def __init__(self, i, lens):\n",
    "        \"\"\"\n",
    "        记录文本对的序号和长度信息\n",
    "        输入：\n",
    "            - i (int): 文本对的序号。\n",
    "            - lens (list): 文本对源语言和目标语言的长度\n",
    "        \"\"\"\n",
    "        self.i = i  # 存储文本对的序号\n",
    "        # 加一是考虑填补在文本前后的特殊词元，lens[0]和lens[1]分别表示源语言和目标语言的长度\n",
    "        self.max_len = max(lens[0], lens[1]) + 1  # 计算文本对的最大长度，并加1\n",
    "        self.src_len = lens[0] + 1  # 计算源语言的长度，并加1\n",
    "        self.trg_len = lens[1] + 1  # 计算目标语言的长度，并加1\n",
    "\n",
    "\n",
    "# 批量生成器，根据词元数目的限制控制批次大小，根据传入样本信息，在不超过设定大小情况下，逐步构建批量\n",
    "class TokenBatchCreator:\n",
    "    def __init__(self, batch_size):\n",
    "        \"\"\"\n",
    "        参数:\n",
    "        batch_size (int): 用于限制批量的大小。\n",
    "        功能:\n",
    "        初始化了一个空的批量列表 _batch。\n",
    "        设定了初始的最大长度为 -1。\n",
    "        存储了传入的 batch_size。\n",
    "        \"\"\"\n",
    "        # 之前的batch_size，就是第一个batch内有多少个样本\n",
    "        self._batch = []  # 初始化一个空的批量列表\n",
    "        self.max_len = -1  # 初始化最大长度为 -1\n",
    "        self._batch_size = batch_size  # 存储传入的批量大小限制\n",
    "\n",
    "    def append(self, info: SampleInfo):\n",
    "        \"\"\"\n",
    "        参数:\n",
    "        info (SampleInfo): 文本对的信息。\n",
    "        功能:\n",
    "        接收一个 SampleInfo 对象，并根据其最大长度信息更新当前批量的最大长度。\n",
    "        如果将新的样本加入批量后超过了批量大小限制，它会返回已有的批量并将新的样本加入新的批量。\n",
    "        否则，它会更新最大长度并将样本添加到当前批量中。\n",
    "        \"\"\"\n",
    "        # 更新当前批量的最大长度\n",
    "        cur_len = info.max_len  # 当前样本的长度\n",
    "        max_len = max(self.max_len, cur_len)  # 每来一个样本，更新当前批次的最大长度\n",
    "        # 如果新的样本加入批量后超过大小限制，则将已有的批量返回，新的样本加入新的批量\n",
    "        if max_len * (len(self._batch) + 1) > self._batch_size:\n",
    "            # result,保存当前的batch，并返回,这之前的batch,_batch清空\n",
    "            self._batch, result = [], self._batch  # 清空当前批量，并保存已有的批量\n",
    "            # 超出限制的样本，放入新的batch\n",
    "            self._batch.append(info)  # 将新的样本加入新的批量\n",
    "            # 新样本是batch的第一个，所以它的长度就是当前长度\n",
    "            self.max_len = cur_len  # 更新最大长度为当前样本的长度\n",
    "            return result  # 返回已有的批量\n",
    "        else:\n",
    "            self.max_len = max_len  # 更新最大长度\n",
    "            self._batch.append(info)  # 将样本添加到当前批量中\n",
    "            return None  # 如果没有生成新的批量，则返回None\n",
    "\n",
    "    @property  # 只读属性\n",
    "    def batch(self):\n",
    "        return self._batch  # 返回当前批量"
   ],
   "id": "cec06911fd33187e",
   "outputs": [],
   "execution_count": 354
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:39.887078Z",
     "start_time": "2025-03-24T10:58:39.881149Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from torch.utils.data import BatchSampler\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "class TransformerBatchSampler(BatchSampler):\n",
    "    def __init__(self,\n",
    "                 dataset,\n",
    "                 batch_size,\n",
    "                 shuffle_batch=False,\n",
    "                 clip_last_batch=False,\n",
    "                 seed=0):\n",
    "        \"\"\"\n",
    "        批量采样器\n",
    "        输入:\n",
    "            - dataset: 数据集\n",
    "            - batch_size: 批量大小\n",
    "            - shuffle_batch: 是否对生成的批量进行洗牌\n",
    "            - clip_last_batch: 是否裁剪最后剩下的数据\n",
    "            - seed: 随机数种子\n",
    "        \"\"\"\n",
    "        # 初始化数据集\n",
    "        self._dataset = dataset\n",
    "        # 初始化批量大小\n",
    "        self._batch_size = batch_size\n",
    "        # 初始化是否对批量进行洗牌的标志\n",
    "        self._shuffle_batch = shuffle_batch\n",
    "        # 初始化是否裁剪最后一批数据的标志\n",
    "        self._clip_last_batch = clip_last_batch\n",
    "\n",
    "        # 下面3个是为了随机\n",
    "        # 初始化随机数种子\n",
    "        self._seed = seed\n",
    "        # 初始化随机数生成器\n",
    "        self._random = np.random\n",
    "        # 设置随机数种子\n",
    "        self._random.seed(seed)\n",
    "\n",
    "        # 初始化空列表，存储后续生成的SampleInfo对象\n",
    "        self._sample_infos = []\n",
    "        # 遍历数据集，获取每个样本的长度信息，并创建 SampleInfo 对象，存储在 _sample_infos 列表中\n",
    "        for i, data in enumerate(self._dataset):\n",
    "            # 输入长度和输出长度，[src_len, trg_len]\n",
    "            lens = [len(data[0]), len(data[1])]\n",
    "            # 将样本的索引和长度信息存储在SampleInfo对象中，并添加到_sample_infos列表中\n",
    "            self._sample_infos.append(SampleInfo(i, lens))\n",
    "\n",
    "    # iter()方法返回一个迭代器，用于遍历数据集，返回每个批次的样本索引\n",
    "    def __iter__(self):\n",
    "        \"\"\"\n",
    "        对数据集中样本进行排序\n",
    "        规则：先按源语言长度排序，长度相同按目标语言长度排序。\n",
    "        使用TokenBatchCreator组装批量数据，满足批量大小时返回一个批次的样本信息\n",
    "        如果不裁剪最后一个批次的数据且存在剩余样本，将这些样本组成最后一个批次\n",
    "        如果对批次进行洗牌，对批次进行洗牌\n",
    "        通过迭代器抛出每一个批次的样本在数据集中的索引\n",
    "        \"\"\"\n",
    "        # 排序，先按源语言长度排序，长度相同按目标语言长度排序\n",
    "        infos = sorted(self._sample_infos,\n",
    "                       key=lambda x: (x.src_len, x.trg_len))\n",
    "\n",
    "        # 把样本放入到箱子里，所有装箱后的箱子，每一个箱子都放入batch_infos\n",
    "        batch_infos = []\n",
    "        # 初始化批量生成器\n",
    "        batch_creator = TokenBatchCreator(self._batch_size)\n",
    "        # 遍历排序后的样本信息\n",
    "        for info in infos:\n",
    "            # 将样本信息添加到批量生成器中，如果满足批量大小，则返回一个批次\n",
    "            batch = batch_creator.append(info)\n",
    "            if batch is not None:\n",
    "                # 将生成的批次添加到batch_infos列表中\n",
    "                batch_infos.append(batch)\n",
    "\n",
    "        # 如果不裁剪最后一个批次的数据且存在剩余样本，将这些样本组成最后一个批次\n",
    "        if not self._clip_last_batch and len(batch_creator.batch) != 0:\n",
    "            batch_infos.append(batch_creator.batch)\n",
    "\n",
    "        # 如果需要对批次进行洗牌，则对batch_infos进行洗牌\n",
    "        if self._shuffle_batch:\n",
    "            self._random.shuffle(batch_infos)\n",
    "\n",
    "        # 记录批量的数量\n",
    "        self.batch_number = len(batch_infos)\n",
    "\n",
    "        # 遍历每个批次，生成该批次的样本索引\n",
    "        for batch in batch_infos:\n",
    "            batch_indices = [info.i for info in batch]\n",
    "            # 通过yield返回该批次的样本索引，yield:返回一个生成器对象，可以迭代\n",
    "            yield batch_indices\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"\n",
    "        返回批量的数量\n",
    "        \"\"\"\n",
    "        # 如果已经计算过批量数量，则直接返回\n",
    "        if hasattr(self, \"batch_number\"):\n",
    "            return self.batch_number\n",
    "        # 否则计算批量数量并返回\n",
    "        batch_number = (len(self._dataset) +\n",
    "                        self._batch_size) // self._batch_size\n",
    "        return batch_number\n"
   ],
   "id": "87136a5120eb02",
   "outputs": [],
   "execution_count": 355
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# DataLoader",
   "id": "83d9c2268ff5be9f"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:39.908745Z",
     "start_time": "2025-03-24T10:58:39.904739Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def collate_fct(batch, tokenizer):\n",
    "    # 对于每个批次中的样本对，将源语言句子按空格分割成单词列表\n",
    "    src_words = [pair[0].split() for pair in batch]\n",
    "    # 对于每个批次中的样本对，将目标语言句子按空格分割成单词列表\n",
    "    trg_words = [pair[1].split() for pair in batch]\n",
    "\n",
    "    # 使用源语言的tokenizer将源语言单词列表编码为模型输入，并返回输入及其掩码\n",
    "    encoder_inputs, encoder_inputs_mask = tokenizer.encode(\n",
    "        src_words, padding_first=True, add_bos=True, add_eos=True, return_mask=True\n",
    "    )\n",
    "\n",
    "    # 使用目标语言的tokenizer将目标语言单词列表编码为解码器输入，不返回掩码\n",
    "    decoder_inputs = tokenizer.encode(\n",
    "        trg_words, padding_first=False, add_bos=True, add_eos=False, return_mask=False,\n",
    "    )\n",
    "\n",
    "    # 使用目标语言的tokenizer将目标语言单词列表编码为解码器标签，并返回标签及其掩码\n",
    "    decoder_labels, decoder_labels_mask = tokenizer.encode(\n",
    "        trg_words, padding_first=False, add_bos=False, add_eos=True, return_mask=True\n",
    "    )\n",
    "\n",
    "    # 将编码后的输入和标签及其对应的掩码移动到指定设备上，并以字典形式返回\n",
    "    return {\n",
    "        \"encoder_inputs\": encoder_inputs.to(device=device),\n",
    "        \"encoder_inputs_mask\": encoder_inputs_mask.to(device=device),\n",
    "        \"decoder_inputs\": decoder_inputs.to(device=device),\n",
    "        \"decoder_labels\": decoder_labels.to(device=device),\n",
    "        \"decoder_labels_mask\": decoder_labels_mask.to(device=device),\n",
    "    }\n",
    "\n",
    "#当返回的数据较多时，用dict返回比较合理"
   ],
   "id": "907e7a2051c9d78e",
   "outputs": [],
   "execution_count": 356
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.092411Z",
     "start_time": "2025-03-24T10:58:40.041088Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from functools import partial\n",
    "\n",
    "sampler = TransformerBatchSampler(train_ds, batch_size=256, shuffle_batch=True)\n",
    "\n",
    "#为什么这里每个批量的样本对数目不一样呢？长度*batch_number>4096的时候，就会返回上一个batch，然后新的样本加入新的batch,具体要看TokenBatchCreator的44行\n",
    "sample_dl = DataLoader(train_ds, batch_sampler=sampler,\n",
    "                       collate_fn=partial(collate_fct, tokenizer=tokenizer))  #partial函数，固定collate_fct的tokenizer参数\n",
    "\n",
    "# 外层循环是拿每个batch，内层循环是拿每个batch里面是一个字典\n",
    "# 输出是每个batch的encoder_inputs,encoder_inputs_mask,decoder_inputs,decoder_labels,decoder_labels_mask\n",
    "for batch in sample_dl:\n",
    "    for key, value in batch.items():\n",
    "        print(key)\n",
    "        print(value)\n",
    "    break"
   ],
   "id": "8af2e4bfa5d41759",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "encoder_inputs\n",
      "tensor([[   1,    7,   17,    6,    8,   89,  157,   73,    7,  526,  246,    9,\n",
      "           14,   42,  248,  120, 1206, 1168, 8998, 4504,    4,    3],\n",
      "        [   0,    0,    0,    0,    0,    1,  493,    6,    8, 1758, 1149, 1985,\n",
      "         2216, 2227,  179,   31, 4477, 2616,   59, 1471,    4,    3]],\n",
      "       device='cuda:0')\n",
      "encoder_inputs_mask\n",
      "tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],\n",
      "       device='cuda:0')\n",
      "decoder_inputs\n",
      "tensor([[    1,     5,    16,     6,     5,    50,    41,    18,    77,     5,\n",
      "           379,   665, 17798,   106,   433,     4],\n",
      "        [    1,   409,     6,     5,  3889,   912,   834,   106,     5,   462,\n",
      "          3955,     4,     0,     0,     0,     0]], device='cuda:0')\n",
      "decoder_labels\n",
      "tensor([[    5,    16,     6,     5,    50,    41,    18,    77,     5,   379,\n",
      "           665, 17798,   106,   433,     4,     3],\n",
      "        [  409,     6,     5,  3889,   912,   834,   106,     5,   462,  3955,\n",
      "             4,     3,     0,     0,     0,     0]], device='cuda:0')\n",
      "decoder_labels_mask\n",
      "tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1]], device='cuda:0')\n"
     ]
    }
   ],
   "execution_count": 357
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "# 定义模型\n",
    "\n",
    "- Transformer模型由Embedding、Transformer-Block组成\n",
    "- Embedding包括：\n",
    "    - WordEmbedding\n",
    "    - PositionEmbedding\n",
    "- Transformer-Block包括：\n",
    "    - Self-Attention\n",
    "    - Cross-Attention\n",
    "    - MLP"
   ],
   "id": "e06f38adc0998bfa"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## Embedding",
   "id": "218b5d3b805bcc98"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "核心原理总结\n",
    "\n",
    "## 双编码组合：\n",
    "\n",
    "词嵌入：学习词语的语义信息（可训练参数）\n",
    "\n",
    "位置编码：通过数学公式编码绝对位置信息（固定参数）\n",
    "\n",
    "## 位置编码设计：\n",
    "\n",
    "使用不同频率的正弦/余弦函数组合\n",
    "\n",
    "### 优势：\n",
    "\n",
    "可以扩展到任意长度的序列\n",
    "\n",
    "允许模型学习相对位置关系\n",
    "\n",
    "计算高效（预计算后查表）\n",
    "\n",
    "## 信息融合方式：\n",
    "\n",
    "简单相加：词向量和位置编码在同一空间维度\n",
    "\n",
    "相当于给每个词语的表示\"打上位置标记\"\n",
    "\n",
    "## 正则化机制：\n",
    "\n",
    "Dropout层增强模型泛化能力\n",
    "\n",
    "防止对特定位置的过拟合\n",
    "\n",
    "## 可视化验证：\n",
    "\n",
    "通过颜色分布检查编码模式\n",
    "\n",
    "理想情况应呈现规律性的条纹模式"
   ],
   "id": "7b7743dd70766dde"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.213356Z",
     "start_time": "2025-03-24T10:58:40.093414Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class TransformerEmbedding(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        \"\"\"Transformer的联合嵌入模块（词向量 + 位置编码）\n",
    "        \n",
    "        参数：\n",
    "            config - 配置字典，包含：\n",
    "                vocab_size: 词表大小\n",
    "                d_model: 隐藏层维度\n",
    "                pad_idx: 填充符索引\n",
    "                dropout: dropout概率\n",
    "                max_length: 最大序列长度\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "\n",
    "        # 初始化配置参数\n",
    "        self.vocab_size = config[\"vocab_size\"]\n",
    "        self.hidden_size = config[\"d_model\"]  # 词向量维度\n",
    "        self.pad_idx = config[\"pad_idx\"]\n",
    "        dropout_rate = config[\"dropout\"]\n",
    "        self.max_length = config[\"max_length\"]\n",
    "\n",
    "        # 词嵌入层（可训练）\n",
    "        self.word_embedding = nn.Embedding(\n",
    "            self.vocab_size,\n",
    "            self.hidden_size,\n",
    "            padding_idx=self.pad_idx  # 填充位置置零\n",
    "        )\n",
    "\n",
    "        # 位置嵌入层（固定编码）\n",
    "        # 位置嵌入层的权重通过get_positional_encoding函数计算得到\n",
    "        # 位置嵌入层的权重不参与训练,固定编码\n",
    "        # max_length是最大序列长度，hidden_size是词向量维度\n",
    "        self.pos_embedding = nn.Embedding(\n",
    "            self.max_length,\n",
    "            self.hidden_size,\n",
    "            _weight=self.get_positional_encoding(  # 使用预计算的位置编码\n",
    "                self.max_length, self.hidden_size\n",
    "            ),\n",
    "        )\n",
    "        self.pos_embedding.weight.requires_grad_(False)  # 冻结位置编码参数\n",
    "\n",
    "        # 正则化组件\n",
    "        self.dropout = nn.Dropout(dropout_rate)\n",
    "\n",
    "    def get_word_embedding_weights(self):\n",
    "        \"\"\"\n",
    "        获取词嵌入权重矩阵（用于后续权重共享）\n",
    "        矩阵大小：[max_length, hidden_size]，根据seq_len截取矩阵部分\n",
    "        \"\"\"\n",
    "        return self.word_embedding.weight\n",
    "\n",
    "    @classmethod\n",
    "    def get_positional_encoding(cls, max_length, hidden_size):\n",
    "        \"\"\"生成Transformer的位置编码（固定公式计算）\n",
    "        \n",
    "        原理：\n",
    "            使用不同频率的正弦/余弦函数编码位置信息\n",
    "            公式：PE(pos,2i)=sin(pos/10000^(2i/d_model))\n",
    "                 PE(pos,2i+1)=cos(pos/10000^(2i/d_model))\n",
    "        \"\"\"\n",
    "        # 生成一个全0矩阵\n",
    "        pe = torch.zeros(max_length, hidden_size)\n",
    "\n",
    "        # 生成位置坐标（形状：[max_length, 1]）\n",
    "        # torch.arange(0, max_length) 生成一维张量，形状：(max_length,)\n",
    "        # unsqueeze(1) 在第1维度（列维度）增加一个维度，新形状：(max_length, 1)\n",
    "        # 后续要与形状为 (1, hidden_size//2) 的 div_term 相乘，通过增加维度实现广播：(max_length,1) * (1, d/2) => (max_length, d/2)\n",
    "        position = torch.arange(0, max_length).unsqueeze(1)\n",
    "\n",
    "        # 计算频率项（指数衰减项）\n",
    "        div_term = torch.exp(\n",
    "            torch.arange(0, hidden_size, 2) *\n",
    "            -(torch.log(torch.tensor([10000.0])) / hidden_size)\n",
    "        )\n",
    "\n",
    "        # 交替填充正弦/余弦值\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)  # 偶数维度\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)  # 奇数维度\n",
    "        return pe\n",
    "\n",
    "    def forward(self, input_ids):\n",
    "        \"\"\"前向传播过程\n",
    "        输入形状：[batch_size, seq_len]\n",
    "        输出形状：[batch_size, seq_len, d_model]\n",
    "        \"\"\"\n",
    "        # 验证输入长度\n",
    "        seq_len = input_ids.shape[1]\n",
    "        assert seq_len <= self.max_length, \\\n",
    "            f\"输入序列长度应≤{self.max_length}，实际得到{seq_len}\"\n",
    "\n",
    "        # 生成位置索引：[0,1,2,...,seq_len-1]\n",
    "        position_ids = torch.arange(seq_len, dtype=torch.long, device=input_ids.device)\n",
    "        # 扩展为批量形式：[1, seq_len] -> [batch_size, seq_len]\n",
    "        position_ids = position_ids.unsqueeze(0).expand_as(input_ids)\n",
    "\n",
    "        # 获取两种嵌入\n",
    "        word_embeds = self.word_embedding(input_ids)  # 词义信息\n",
    "        pos_embeds = self.pos_embedding(position_ids)  # 位置信息\n",
    "\n",
    "        # 合并信息并正则化\n",
    "        embeds = word_embeds + pos_embeds  # 逐元素相加\n",
    "        embeds = self.dropout(embeds)  # 防止过拟合\n",
    "        return embeds\n",
    "\n",
    "\n",
    "def plot_position_embedding(position_embedding):\n",
    "    \"\"\"可视化位置编码矩阵\"\"\"\n",
    "    plt.pcolormesh(position_embedding)\n",
    "    plt.xlabel('Depth')  # 隐藏层维度\n",
    "    plt.ylabel('Position')  # 序列位置\n",
    "    plt.colorbar()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# 示例使用（生成并可视化位置编码）\n",
    "position_embedding = TransformerEmbedding.get_positional_encoding(64, 128)\n",
    "plot_position_embedding(position_embedding)\n"
   ],
   "id": "69fad20a77cc4f6e",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAG2CAYAAAC3VWZSAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAhQJJREFUeJztnQd4VNXWhtf0mfRKQg+9dwRB9CKgoFhQr4qiIKJYsAAqir8K2LCLKIoNlKteEK+iWFAEwasgVVSKCEiH0NPbZOb8zz7c7LVPmAnJJGEyme/1ObJmn34yyazZe337M2maphEAAAAAQIhgDvYFAAAAAABUBCQvAAAAAAgpkLwAAAAAIKRA8gIAAACAkALJCwAAAABCCiQvAAAAAAgpkLwAAAAAIKRA8gIAAACAkALJCwAAAABCCiQvAAAAAAgpkLwAAAAAYcCPP/5Il156KdWrV49MJhMtWLDgtPssW7aMunbtSg6Hg5o3b07vvffeKdvMmDGD0tLSyOl0Us+ePWn16tVU3SB5AQAAAMKA3Nxc6tSpk55slIedO3fS4MGD6fzzz6cNGzbQ2LFj6ZZbbqFvv/1WbjNv3jwaP348TZo0idavX68ff+DAgXT48OFqvBMiE4wZAQAAgPDCZDLRZ599RkOGDPG7zYMPPkhfffUVbdy4UbYNHTqUMjIyaNGiRfpr0dNy1lln0Wuvvaa/9nq91LBhQ7r77rvpoYceqrbrt1ItRzzIAwcOUHR0tP7DAgAAAPwhvs9nZ2frQytmc/UNThQUFFBRUVGVXK+p1GebGOIRS2VZuXIlDRgwwNAmelVED4xAXP+6deto4sSJcr14ZmIfsW91UuuTF5G4iCwQAAAAKC979+6lBg0aVFvi0qRxFKUf9lT6WFFRUZSTk2NoE0M4kydPrvSx09PTKSUlxdAmXmdlZVF+fj6dOHGCPB6Pz23+/PNPqk5qffIielwEDV55kMwuB/3a919y3TU9zpXxiUGtZJzZlLPYhk+vMhxv9+SeMm48mddZWjWXcXGCS8Zmt1fGN771jYwvj8qQceelI2Tc6vGDMs7tVN9w7ldf4HHKqQcHyfjYAD7Wrqd6yPjegV/JeOFIvtfcJ/Jl/FHrj2V8S7vefJx32hjO7VodKePUd3+V8ePrf5bx/RNul/HTz70t47nHzpbxol86yfi3K2bJ+J9XXCXjge/9Yjj312clyPjvZ8+SccvX02Vc0CyJ76+uTcYJX/Iv0F9TWsi42dh1Mo79LpH3vd5warphMW/33uhLZDxjDt/fjVPulfHix9+Rcc93b5XxmlHc3vHrm2S86qK3ZNx//TDDued1mi3jB3Zz1+59DXi8+bOMbny+qB0yTi+OI1+kWvm9siGvsYwviOZuYcHcY/w+vy15uYyfPXCRjKc0+FLGY3fwz++t5v+R8Q1brpPx/LbzZHzp7zfI+JuOHxnO3X/djTJe1v0DGfdZNVzGK3py+9k/c/vqc/j3u/ty/r1Sf+87/8Dbbzh/juHcnZfwPhv6v8/ti/ln9vsF3N7hW97+j4Hc3vEbZfuL3vP5s//9YmPhY8evRvK6wfyz77hQab+0HO1fKO2XKe2f38ztl/PvnqDTAl7325BZp2//7Gafv8cVba/KYwVy7s7Kug3/W5eV46XGXXfJz47qQPRYiMRl97o0iokOvHcnK9tLjbvt0hOtmJgY2V4VvS41nVqfvJR0p4nExRzhNLxRrCa7jC12J8dOTl6sJv4g1I/jdPpcZ7EobxYrb2PWOHmJiLbIOCaKr8PsUo5p5uNYbdwuiFKu3ZZl93kd6vW5ovjHa1WuzxrJ1xRteB7KcSKM57Y4fN+3ek3q9art9kKbz3uNUZ6Hen3qdZ9yXf6elfLMLXabz5+xYV/lmLZI5VmW+jui/szUc6jPTX3vqO8v9Zmp92p8Bsr2EcY/OOo51GuMVNodxTaf1+p0+/7VjrDxNg4z7xup7CuwF9p9v++U6zD87CMdvt9TftrVey39x9vfOovynlTbzRVt9/P8y1rn731bVe1n4hw4t49zO/2vOxNlBlHRJn0JFC+d3FckLmryUlWkpqbSoUOHDG3itTiXy+Uii8WiL762EftWJ1AbAQAAAEHAo3krvVQnvXr1oiVLlhjaFi9erLcL7HY7devWzbCNqDMVr0u2qS5qfc8LAAAAUBPxkqYvldm/IojamO3btxuk0EICnZCQQI0aNdILb/fv309z5pwcTr399tt1FdGECRPo5ptvpqVLl9LHH3+sK5BKEDLpESNGUPfu3alHjx40bdo0XZI9ciQPX1YHSF4AAACAMGDt2rX6nC1q4iEQyYeYfO7gwYO0Z88eub5JkyZ6ojJu3Dh65ZVX9ALmd955R1cclXDttdfSkSNH6LHHHtMLfDt37qzLqEsX8VY1YZO8tHgln6wWL83s1ki2HRzeVsapb66Xcb9VXND62/tNDMe56IK1Mt4+gwtqD/bjgtGUWVzQSsq46cs7WHLWos2HMr6w7WY+ZksulI3cctRw7iV5XFR8adJvMn7f0VLGUbv4fGe5dsr4w4ZcbLr/EI/zxrfn4mLNw5XvnsPcLijkmlnyKvK+ZAvvUxjHY8bpxTz+Ws95QsbWHN8jlaZ8PubRYv+FcpY8ZX+HUnNUoFy7nesyNLebt7f77mKNsPK5c6wRhnVOE++v2ZTaj/+NNQu8fn6LNIu/dv625FGmWTKbjdenvrIq6zyaUgdi4v29aruyt1uz+mz3anwPZsPZxDl8j8Or+1jK8a1P83OcgPB3uoqeo6bObFXR66qp9wHKjVf/r3L7V4S+ffvq0mp/+Jo9V+zz66/KZ5oP7rrrLn05k4RN8gIAAADUJMSXF/ULTCD7hyso2AUAAABASIGeFwAAACAMCnZrE0heAAAAgCAgkg8PkpeAwLARAAAAAEKKsOl50bbtIs1ko7feulS2/WO4ohxaUEfGU5I/l3GXa84xHOeD1H/L+ILB98m4sF8Wb/Q2K1+87mIZZ/zC53i/Dh93VPKPMh7dobOMU1ccMJz7q0MdZDyj6XwZ/yuZJwOK28nna2rlrDy7Af+ozQeV2SfV/NWkzNx61JjXFib5rmqPMvFxC+NY9XHAHS/jBrbjMrYpFhxuja+VCgpleLiolNpIUfxYC7hZU9RG5kI+lkedqNbD122yeX3eq8uimKNZov2rjSyK0kZRkXmtqmrHfFq1kUlRG6nfnIQaTsWjKJqsJkUlpJzDalJUVkq7WVEhub0W3+qkMr67qMol9Viq2si4vfoMTk9ZdYZVqlCqKH6uK4zrIkE1gmGjwAmb5AUAAACoSUBtFDgYNgIAAABASIGeFwAAACAIiMHgyk1SF74geQEAAACCgKeSaiMPal5qP4dHdCGL3Ul13+Rpjl+ZsEbGbUbdIeNNSpFtnys3GI5jU0bavJdwIeqjrb6T8XudB8vYejRbxnV/5sLQL5p3lPGL/fk6MtvxuesUKtWp4rr+ZjuCRi25sLS4IVsTuHZlyDjGzFP85zbg40Qc5ILIQo0LUs1OrnR1HTGcmopacUGtycIFoBFmnoq/MI6331fEfgLnR7P9gS2XtylUC3YL+fhHCoxFsyYz2wtY8rjd61SKkAuUgl2+JIPlgdXu8XkPUYo9AFmtfgt2vQZ7gNMX5qrthmm8zZrPPz5qMa2+j/LSalYLc/nnZ1MKdt3KCe2mYp/bqzYAqgWARSkILssGwF9hrr8iW38FvoEU5VZ4n0r+XTcUs/s9R0Wv6QwUIwez4BlUCI92cqnM/uEKal4AAAAAEFIEPXkR9ts33HADJSYmksvlog4dOujOlyUIEynhVlm3bl19/YABA2jbtm1BvWYAAACgqmpeKrOEK0FNXk6cOEHnnHMO2Ww2+uabb2jz5s304osvUnw8zxHy3HPP0fTp02nmzJm0atUqioyM1O24CwqMQyoAAABAKOElkz6sG+jiVYaEw42g1rw8++yz1LBhQ5o9e7Zsa9KkiaHXZdq0afTII4/Q5ZdfrrfNmTOHUlJSaMGCBTR06NCgXDcAAAAAwrTn5YsvvqDu3bvT1VdfTXXq1KEuXbrQ22+/Ldfv3LmT0tPT9aGiEmJjY6lnz560cuVKn8csLCykrKwswwIAAADUNERRfmWXcCWoPS9///03vfHGGzR+/Hh6+OGHac2aNXTPPfeQ3W6nESNG6ImLQPS0qIjXJetKM3XqVJoyZcop7VfespScUTb6cSlPvz8/J1bGd1+9UMY3rL9Zxmt6cq+QYNJhnor/1XZsFdDDwaOPT/ZjtUzM7kgZx3/HtTpRHVrJuLAfK1o6tN4jY3cCD58JIv5iGY17IKtJsptEyDhu0R6fCpeiBqzmSdjMxznk4eE3c3QUn+uwcTTVFs/z+pscDp+KjKI4/k3ak89qo+Q4lhjZcnibPEVtpCn2AEcLjPdtt+b4tAfwOvjta8niFV6Hb7WRzcbnM5m5u9VlVtVGRumQXVHzGG0AVHsAwy5Ku1YhewCL2fjMiww2AF6fqiJVJeRRp/Q3KITMPrf3pyg6eV0VUw/5216dANRSzi5uVVXkT/GjPv+KTzIavl3toGZRMvxTmf3DlaD2vHi9XuratSs9/fTTeq/L6NGj6dZbb9XrWwJl4sSJlJmZKZe9e/dW6TUDAAAAIIyTF6Egatu2raGtTZs2tGfPyd6D1NRU/d9Dhw4ZthGvS9aVxuFwUExMjGEBAAAAahqVKdb1VLLXJtQJavIilEZbt241tP3111/UuHFjWbwrkpQlS5bI9aKGRaiOevXi4RsAAAAg1BDDsJVdwpWg1ryMGzeOevfurQ8bXXPNNbR69Wp666239EVgMplo7Nix9OSTT1KLFi30ZObRRx+levXq0ZAhQ4J56QAAAAAIx+TlrLPOos8++0yvU3n88cf15ERIo4cNGya3mTBhAuXm5ur1MBkZGdSnTx9atGgROZ3OYF46AAAAUClQsBvC3kaXXHKJvvhD9L6IxEYslWFc/N8UE22hNyf8Q7Y9vOA6GW+7gYuEP/qI1UIHurMSSPDZ9z1l/Nww9kk65s2XcUK/gzLe/ycrpaL/fUzGddazOma54uVzTSr7HP2r6cWGc8f/xcqXHcWszslK4zdw9IlMGR/1sBFQo/p8btsBVgJtcyuGRPGsvnIe4eMLGsSyj1NWFCuoVLxx/KwO5HGtUaKiorFnszRECUkrZiVQZgF7Mgnq2PhtauXHTB4nq25sRXxuj6I2Io3P7bQrXkqKt1G0pcCv2sipeAR5baoXkKJ2sZ7e28ijXIdZURt5FKmMzcI/X/18Spew0dtI8VhS1VB+PI9UFZJFUaCpKiTzKb5KfryNlHNY/PzdVJ9NID5F/sRDFVcVBZFQutZaTqm3do1C/C6rv88V3z98CXryAgAAAIQjWiXrVrQwrnkJurcRAAAAAEBFQM8LAAAAEARQ8xI4SF4AAACAICDq0dSatIrvT2ELho0AAAAAEFKETc/L0B0DyBZpp2UDpsm2O/qPkPGTg1rLOPrr32V8/cibDMdJW8jKlEVXskfQz9ntZfx4889lPDv6XBkfTanDB9rMHkTvHjxPxjMaL5Dx9PbsNSRIXJ8h46W57I2U20RRRCmqlm1u9jw6O2mXjH8/xNf9W0EjGRcn8vlsx9iPSNAyimc5XhvdQsb5GquSXLH8bA7n8LGiTPw2s+VwffxxRRakuVnVk5OnyoWI6thtMrbmK0odp5J7F7I/kYdvz4DLxs/JpKiNIsx8D5rNqDayqV5ABm8jPrfXuAsfy+rbw8hsUY6pbK/6Dun3YVAP+fEw8tPuMLl9KoRUz6OyupxVJVJ5qKjnUUXVSWWiVWExo799wvgbLqg+xO+HtxJ9CN4wfmOGTfICAAAA1CRQ8xI4GDYCAAAAQEiBnhcAAAAgJAt2NQpXkLwAAAAAQat5CXzox4thIwAAAACA0CBsel4yX21AVpuTMqfzLXv+ZgXOvA/Pl3FD2yYZm+YlGY5jWbFWxuN+vUbGhUfZj+eJyzbKOLru9zK+76wxMnZ8uVrG635jv6Q6TVmlc5wFTDqJC9gz6avDHfh6Gx/l64vi/VflN5Px2VHbZfzHsWQ+d2ZjGRcks8oname64dzNnaw2Wh3XWcbZXla1pMRky3jPYfZPcihqI2suq4qOedkjSfOwCqkozygXMtlUtRGra4odSu7tVtQ1Tt9dqRG2Ip/eRhHmojLURppPbyMzlcPbyM9XA1Vt5FHUAvZS3kaq54nqbaSqE1QPI7dipmRRrtvgbaSok4oVmZTqeaSfQ1HdmE2nVxVpFWwva0r0ih6rwoRvTzuoYXgr6W3kDeM3M3peAAAAgCDWvFRmCYQZM2ZQWloaOZ1O6tmzJ61ezV+mS9O3b1/dILn0MnjwYLnNTTfddMr6QYMGUXUSNj0vAAAAQE3reTnT87zMmzePxo8fTzNnztQTl2nTptHAgQNp69atVKeOMhfZ//j000+pqIh7p48dO0adOnWiq6++2rCdSFZmz54tXzscxvm6qhr0vAAAAABhwksvvUS33norjRw5ktq2basnMRERETRr1iyf2yckJFBqaqpcFi9erG9fOnkRyYq6XXx8fLXeB5IXAAAAIAh4NFOlF0FWVpZhKSzkWcNVRA/KunXraMCAAbLNbDbrr1euXEnl4d1336WhQ4dSZCTXLAqWLVum99y0atWK7rjjDr2HpjoJm2Ej59fryGqy0ZD+Y2VbvSG8vtHsbTI+OqStjBMXbDYeKJKn3I/9IsrntPU7Ls6RcTc7b3OwFz/uZsu4PXE9F06euDyP29tyIa7Ak8H2ANt28BT9N3bnN93aFG5feYKLiC9v9IeMvZlcWLv5aJqMLXX4OiKz+R4ETW2HZVyY6JTxcS8XUTaM4uvbtT1Vxja1YDeHux/T3XE+bQ0ot9R8+w67z4LdgkQ+rlakFOzajcWnJUQpBbsFVt7XaeZ9Nasxn7cp3bKqPYDFpNgD+PstMtgD8DVZzOqU/srmpewB1KJWtTC3SC3MNUz3rxbyFvsu5FXvRyk6VttLr/N3TRZlG7/2AH6Lb6laUH8uZwKlLrrGE9C1htD9hSKeShbsev73A2rYsKGhfdKkSTR58uRTtj969Ch5PB5KSUkxtIvXf/7552nPJ2pjNm7cqCcwpYeMrrzySmrSpAnt2LGDHn74Ybrooov0hMiiiCOqkrBJXgAAAIDayN69eykmJqba601E0tKhQwfq0aOHoV30xJQg1nfs2JGaNWum98b079+/Wq4Fw0YAAABAEBAGqJVdBCJxURd/yUtSUpLeE3LoEE99IRCvRZ1KWeTm5tLcuXNp1KhRdDqaNm2qn2v7dp6io6pB8gIAAAAEcdioMktFsNvt1K1bN1qyZIls83q9+utevXqVue/8+fP1WpobbrjhtOfZt2+fXvNSt25dqi6QvAAAAABhwvjx4+ntt9+m999/n7Zs2aIX14peFaE+EgwfPpwmTpzoc8hoyJAhlJiYaGjPycmhBx54gH755RfatWuXnghdfvnl1Lx5c12CXV2g5gUAAAAIAqJEv0QxFOj+FeXaa6+lI0eO0GOPPUbp6enUuXNnWrRokSzi3bNnj65AUhFzwPz000/03XffnXI8MQz1+++/68lQRkYG1atXjy688EJ64oknqnWul7BJXooGdCWvzUmtX9wn27LeZhWL9m2+jC1Dj3D7fFaiCHIG8bT8id/u4BXKD/vx/RfL+NkGX8u40dl7+bgteVr+pF8zZfxTAdsR/LPRr4Zzf2+JlXHkXzxl/nl9uUr850ZsNfB3OiuaGjThN5G3sEDGGelc5BXJrgHkzefnIahvzZVxYQK/bfYX8/5pESyNW5nlu1PPnMeKn0PKviqWvFL7Kr8A1nxW3RQ7+RlQMatrNIfvX+kIq6o24p99pJllhZrVWBlvNymKGuV0Kn7tASzqFP0cWxV7APVKS9sDqKoim6JQKhnnPlVVZPU53b8/xU9ZfzSNqiJFoVSOqftV6wQVf+2nHqx8m/k6d2WOc0bQAlhXE+8D1JBJ6swB7XfXXXfpiy9EkW1phPxZ8yMRdLlc9O2339KZBsNGAAAAAAgpwqbnBQAAAKhJVMafSFCZfUMdJC8AAABAEBCTQfqbELK8+4crSF4AAACAIICel8AJ3zsHAAAAQEgSNj0v1nsPkTXSQdqV7OvzRbtFMr7g+vtkPK/d8zIefim3Cw5exOqOiM+O+vTmWfFfnuxn9kXpMr4vjWVmj3TjWQrr/Pt3Gc89zNMuT67/peHcPyRdIeP4v1iZ0tmRJeOsJqyice9jeYzD5Fsq4zzIb4GCZK4m1zxG5UuCmbfLT+Scd6+bNf9NHKzSsmUrKh1VU5PHKqYDBarrKCuBrHnGrlBNURVZ8vn5F7PFEmlubjfblWtXvG6irayyOm5ljyqnic/ttZfyNlLVRoq3kVnJ+71+rDtMVlVV5Ftt5FG6fU/xNlLOYVW8jdSJqcyKYY2qHrIo7epxhLZBtmu+j1Ner6LyfPPx52HkVyFU5sGqv4u8ujyXAKgebyMzhSthk7wAAAAANQnxZcDfF4Xy7h+uhG/aBgAAAICQBD0vAAAAQBAQQ7qVGfrxhnH/A5IXAAAAIAioztCB7h+uhO+dAwAAACAkCZuel89afUMx0WbqcPcY2XbAw2qjdiM3ybiehdUtWUNZnSR4rM33Mp7XZYCMzcdY8dNwCStfZjXoLePN/9gs4zu7snok8e0cGa/8q7uMm6WxIkZQ3ITtxaO2sx9SojmSrzeNC7gi9nJcqLFHk9nlkrHrEB/f3YzVOCaLUUITZWZ/oYIEbt9ZyIZI50Wxx5KdH4fh3JTP50gvYG8jk+WEjBUbJR2vi38epgJ+th7F80tVR1kdHp/3EaV4G5GNjxmhqo1spdRGqqrIn4eR0m5QVpkVJZCqNjL4FPHmVrNR4aUqkWyK2siteB7ZFW+jTM3lU1VkVCF5T+tfVHqduRwqofJ4HpWXCu9TUYWQsr2qGit7n4pe0xkopAzjYs3agvgdV3/PA9k/XAmb5AUAAACoSWDYKHDC984BAAAAEJKg5wUAAAAIAmIwuHLDRuELkhcAAAAgCGDYKHDCJnl5O7MROT1Wmjz8I9l2+U93ynhL33dkPGb/+TL+qMu7huO0s3OV6LOXxco4ZicXnyYu4MLcqBbtZFx4HhdXntVhh4xzkpN4+43K9P4XGPPqzBZcwJuwcLfPIlF3Ey6ITfySj7XPUyhjcxxfd9RBPocrKctnUa/+WhlhLErk8+3KY3uA6+J4f3sWV0Vme7lgVyvg6ziSz5W/DisXLVvzDKcmj5PfpvZMthfwOH0X7Nrt/JxNZv5WE2XhZ0NWLnp1mot9WgAIzMq3Iq9vhwXyWn1XjJr92APYLHytRQYLAKM9gFqYqxbaqmZsZuW46h+y8hTmluVI69cewM8+6rT6lnJ8k1SLcssqmlWff8Wn7g/fYkYQGsCYMXDC984BAAAAEJKETc8LAAAAUJPQyFRmD2h59g9XkLwAAAAAQQDDRoETvncOAAAAgJAEPS8AAABAEBCF8f6K48u7f7gSNsnLnNkDyeJw0soHpsm2t97k9UvOZiXPyv90kvGMe340HOcvN6tdegxkS4GftjWTcdwcthRIXcHT+H+S00jGt9RdLuPn2t0o48SNrMz5XZnNXpDRgt+osRkZMt5XzPPpt290QMYFe1NlvLmojoy1RFYbuQ6yAqdBPHsFpEdHGc6tKpq88XyNe3PiZJygTIdvz+Ltj3sVxUgR73s8j595PTtLeWx5RlmJx6lYFShqJdUegDQ+X4RdeXCqPYCiNtJsioJJmS3Bazd2RlpMitrI6JjAx1J+izzKdZgsij2AIpWxKTYA6h8fh4VVT/o+Sseoag+gjpGr7WoXskVVoCmqJbNJK589gHIOi/L30bCP+mz8qpD8tPtsLdmHQodQutZajvLWDhk8lXSV9oTx4En43jkAAAAAQpKgJi+TJ08mk8lkWFq3bi3XFxQU0JgxYygxMZGioqLoqquuokOHFCdBAAAAIMSHjSqzhCtB73lp164dHTx4UC4//fSTXDdu3DhauHAhzZ8/n5YvX04HDhygK6+8MqjXCwAAAFQFXt3/vXJLuBL0mher1UqpqVybUUJmZia9++679NFHH1G/fv30ttmzZ1ObNm3ol19+obPPPjsIVwsAAACAYBP0tG3btm1Ur149atq0KQ0bNoz27Nmjt69bt47cbjcNGDBAbiuGlBo1akQrV670e7zCwkLKysoyLAAAAEBNw6OZKr2EK0HteenZsye999571KpVK33IaMqUKXTuuefSxo0bKT09nex2O8XFsZpFkJKSoq/zx9SpU/XjlCb13Q1kNdnpnP7DZVvyj7/KeMyXI2Xcet5+GT86tJvhOOuON5Txv1rOlfG8mPYy/rbtWTLWNrGH0St/sWfSL90+kPH9Xdmkp8GnfO75GT0M59ZasarIpKho1hdyz1W/pK0y/i6dj/tzdgsZF6Wwksix67iM20WxUulgQnfDuXO8rPKJS+DrSM9kT6doMyuG7NmsgjniieR7KGa1UU4OX5/JbvevNnIpMh9FreRx+pYXRCpqI5OV3+IRZkWFZFO8jVTFjq20t5H5tN5GmqIqUj2MLAZvI8ZiVnyKFJWOVbkOgVuRMamqoiLN9z0ZVEXkT1Wkeh75/+7ibyzdn3qoPMdR1UkB4UdNUtFrorK2D0HFCghdIJUO0eTloosuknHHjh31ZKZx48b08ccfk6uUMWB5mThxIo0fP16+Fj0vDRtywgEAAADUBLRKukprmGG3ZiB6WVq2bEnbt2/X62CKioooQ5nPRCDURr5qZEpwOBwUExNjWAAAAABQe6hRyUtOTg7t2LGD6tatS926dSObzUZLliyR67du3arXxPTq1Suo1wkAAABUFjFsXNklXAnqsNH9999Pl156qT5UJGTQkyZNIovFQtdddx3FxsbSqFGj9CGghIQEvQfl7rvv1hMXKI0AAACEOl6tcnUr3jCu0Qpq8rJv3z49UTl27BglJydTnz59dBm0iAUvv/wymc1mfXI6oSIaOHAgvf7668G8ZAAAAACEc/Iydy6rdXzhdDppxowZ+lJZzI0bkNnioKTnlELgXuxh1OrtEzIu3r1Xxl8s6G04jk1RXsffz+Y6o2L/lPF7/bgQOeVPVhu5Vybw9XTjbDu3K/sleaaz4ufLHe0M5+7deKeMD9c5meAJvs/kH+PIJJ7k79sjDWS8+mhjGefXZWWPfR3fd1snK52+SfyH4dzHvey70ySOFUq/bmO/JpeJn4cti1Uw6cXspaR5FGVPriLfcSr75qraHKKiKLNPtZHXn9rIxuf2KB5G0WZ+zl47tztUv59SiiKz0i2rehipaFZVbeT1oypSfJ8sHp8KIZvJeN+qDNLgbeRXPaQqe06vKlK3N5f68mc8x+m/GVZU8VPW9uU5lsVUjhFvLTx9cwK61hC6v9qEt5IFu14U7AIAAADgTCIMTSu7BILoEEhLS9M7CITKd/Xq1X63FdOZlLbxEfupaJpGjz32mF6vKpTCYn42MYdbdYLkBQAAAAgT5s2bp9eSihrT9evXU6dOnfSSjMOHD/vdR9ScqjY+u3fvNqx/7rnnaPr06TRz5kxatWoVRUZG6scU/oTVBZIXAAAAIExm2H3ppZfo1ltvpZEjR1Lbtm31hCMiIoJmzZrldx/R2yKmKClZxGSxaq/LtGnT6JFHHqHLL79cn7Ntzpw5ughnwYIFVF0geQEAAACCWPNSmaUiiLnThPWOarsjRDHidVm2O2IaE6EKFhO+igRl06ZNct3OnTv1We/VYwq1sBiOKuuYlQXJCwAAABDCZJXy8xPqXF8cPXqUPB6PoefkdLY7wr5H9Mp8/vnn9MEHH5DX66XevXvramFByX4VOWatcJU+U2wdG0Nml5Na3LxWtm17j/17Woz8Q8ZaH1YhNZl3xHggjcvyX7qZ1UBjE3j/4n6ZMrZ810TGdVfw+N+60XzIy9v+LuONxawq0TZFG059SacNMn4r7QoZrzjI/k/PpP4oY08OexDtOtBSxlGp3NUYq2zTzMoqooJkVv8I9hezP1GzqKMy/u1EM/KFOYfvda+bVVYqlhyLT7WRNc/o8ZOXzNtpRawk8jqN6pwSYux87gzF2yjSzL/Qmo3zdrviueMt5W2kqlr8eRuRojbyKO8Pq4Wvz62oOexmj29vI6X95D587WZFPeSv3aN8F7Go3kbKOdT2YuVbm9qu7+OnO9qgUFKO669deRyVxq8KKZSUMqF0raDa0YtuKzPPC53ct7QFjqhnmTx5MlUFYm41dWJYkbi0adOG3nzzTXriiScoWIRN8gIAAADUJLRKKIZK9hfs3bvXYIUjbHJ8kZSUpE8EK2x2KmK7oyJmvu/SpYtu4yMo2U8cQ6iN1GN27tyZqgsMGwEAAABBdJWuzCIo7efnL3mx2+269Y5quyOGgcTr8truiGGnP/74QyYqTZo00RMY9Zhi6EqojqrTygc9LwAAAECYMH78eBoxYgR1796devTooSuFcnNzdfWRYPjw4VS/fn2aOnWq/vrxxx/XLXmaN2+uGyU///zzulT6lltukUqksWPH0pNPPkktWrTQk5lHH32U6tWrR0OGDKm2+0DyAgAAAITJDLvXXnstHTlyRJ9UThTUiqGdRYsWyYJbYX4sFEglnDhxQpdWi23j4+P1npsVK1boMusSJkyYoCdAo0eP1hMcYfUjjll6MruqJGySl6/6vk7R0WYaev0Dsu3TvtNkPHbw3TLeczGPQba40/8sge/8cL6MowZwkehj7b6S8QtnXyfjxAWbZTzryHkyvrsOd7c9lHwJb7/RWJB6jvOgjJ9rGSHjzJ1seRDVTXmzaLy/fS93I+bV5apBrZin20+xcGFsXrLxl+Jvdx0Zt3axhYE9UynaVKaqN+XwVPz7C+OVI3FRqi1bKex0sWWBJZetCATFLr52rZjXmRxKgatSWBtt5cLcDJvLZ8Gu165Oy8/X4SlVsGtWRla9/n5bDPYAHNusvgtz1YJdtfjWYTbet1qAq9oD5CmWAnalvVhpNys/C7dXaVetEMooFPS3zl/RrL/C3ICKbCtRwFgeqrKI2P9JztA+IKRRh34C3T8Q7rrrLn3xxbJlywyvhcegWMpC9L6IHhqxnClQ8wIAAACAkCJsel4AAACAmkRl/IkEldk31EHyAgAAAITRsFFtAMNGAAAAAAgp0PMCAAAABAH0vARO2CQvJ7w2cnvN1HHsb7KtuXL3WbdlyfiJlotl/K/uFxuOYzmeLeO0hawOmRbfX8bb+r8r4wd7sRok7n22Dfh+E1sTzGzwk4yLm9eXccyWE4Zzp1qiZJzZgt+0Ubs4LtRYPWR2KUqb/XyczD6sjDIpCqMoM6t68llcpLO9gH0rzo9m1ZQjg7fJ8/LU/ZTHaqN9eaw2Mln4nqzsTEDeCEVtlKccRzwTJ1sTaG5+5jZnsc/7iLEpNuw2ntM/wqRYC9gVJQ+VwwJAnFt5v6jKKlJsADyKZMRq5navoiRRbQBUFZKqKBK4DaoivtdMzeVTVaQ6zFoU2wD1D5zBNsAwpX+pe/WzrqIqJH9UdPuTOwW+vaoaK3ufCl7XmfjwCOMPqNoOkpfAwbARAAAAAEKKsOl5AQAAAGoS6HkJHCQvAAAAQBAQI5uVM2YMX5C8AAAAAEEAPS+Bg5oXAAAAAIQUYdPzcuNXd5DZ5aQdV78p2y7Y8k8Zf935XZ+qnknXcyyI/juat3t7vYwTG3SR8dG+eTIe1O13Ge+uX0/GsetYXZN3AatgjrdjZU3y3B2Gc6tKImqZw8f6N6tPdhbzscwJrPKJ3stqldg6x3mbiAifiozCJKOv0l85rDa6IW6VjB0Z3HGZo/E5tHxW/BzMS5ZxpJ0lRja+BfK4WOZjOcHP7+Q6jjUPK3KcDrdvtZFFObCN3+JOxTtIVRuZlW5bL/9YTsFr43v1KL5RZqvXt7eRha+1SHm2Rm8ji1+1kUcxXTMbVEJmn6qiYsXDyKAq8tMtXaa3kZ99VF8gSzm6u42qJd/fldTnX/oc5SN8v32C0AY9L4ETNskLAAAAUJNA8hI4GDYCAAAAQEiBnhcAAAAgCKDnJXCQvAAAAABBQNSEBTTb9P+ozL6hDoaNAAAAABBShE3PS8vX9pPV7KBxfbrJtry32Ueo4DmWOMzPiZXx+EFfGo7z4Z4e/GIW537JPxyQ8UtHz5HxPXWWynh0j3EyrrOOVTff5yfK+ER7Vo8k5OQYFT+Kr885jf+W8YFdjWS8Ir+pjL2pCTJ27edjtU5go6O/4lNl7FbUQpY67E0k2JnJx0pqzPftyGCFzBEPq120IlY9HctmBVWUneU89hx+5sURio9PYaHh3MWK2ogUlU+kQ/FAsvA1RVtY6aQ5FLWR4g/ksaueQnxubxm/EZqiNlJR1UYeRSpjtyjqJuUbkkNRPRUphkk2pb20EinCXOiz3aJ6GymqG7NJq5AKyVLqC5zBD8lkKocKyff2/oRDFVcUBZlQu95ajPLWDnnE71NlJqnzhrHSLmySFwAAAKAmgZqXwMGwEQAAAABCCvS8AAAAAEEABbuBg+QFAAAACAIYNgqcsEletKwc0kxFtPKVs2Rb3CdrZXzRP++QceExrhD9+9K3DcfpGcGFsvefP0bG9q9Wy/jjlT1l/MyQ32R8sDeP0rV4areM395/nozrtT0kY0uU0ZpgcW5bGV+asIH3P8BT/P9worWM8xoohbKr+Xxdozj+M5m3P+HlQtcGiRmGc+85zAW7USal6DaDp+jf7+FCZ69SXJyf7ZSxyemQsS2Hi03dkVxUSgXGgl2Py3eFXrSDr9dkY3uBaAsXG2s2Pq5TmUpfLdg12APwYU5BqXs12ABYlIJdt9JuN9gD8M5WxR7Aq4zclrYHMBTmKlWKqm2Auo+hyFYp5FXbzaby/eHz943OX7u/Y1X0OCdXVuya/FcFV3B7AM4w6HkJHNS8AAAAACCkCJueFwAAAKAmIXpOKjP0o4VxzwuSFwAAACAIaJWc80ij8AXDRgAAAAAIKdDzAgAAAAQBMUOu+K8y+4crYZO87LmtDVkcTmrw1C+yzdKEp9Wv9yarYMxFrJRZPMD4iC5QpqrffRnHbdbWkXH977l91yXZMm7fk5VKeRms5tn+Gyt+xl7wjYy/adLLcO6vDrLi58NW/5bxzGMnZLxmXxsZuxoq08sv4vN1cOyTcX5dViod8PC9to5j1ZN+r1vqythmUqa0z2DFz66iZJ/T+JsylWfo4gdoy2GlTG5dlvlohUXGX1AXH0slzs7nzrEraiMzt3sd3O5UFTuK2shiMpdLbUQ2RcGjqHmsFtUegDe3K6oit6IQMtoDWPyqjTwGJVKxH3sAxQbAT7u/P3BGdZJxG4NCSVVj+VUPUbWg/mxCaup5rQZeaziPMdRQoDYKHAwbAQAAACCkCJueFwAAAKAmIXoyTZikLiCQvAAAAABBQAy3VkptpFHYgmEjAAAAAIQU6HkBAAAAggAKdgMnbJKX/7tuHkVEW+it76+QbdsvZe+ftP9bwRsrCofbfhpuOM7zvefLeNy538r44z4XyTjmvztk/Nbxc2Q8vsF3Mp4ay35Gyev4DXjpFZtkPLcdH1NwcEeMjFPbsu+RVsz+Qp6dfE+5DXhfbyErcBpaFZVPKr8FthWlyLh95H7DuX840ZV8YcrKk/GugiSf29iy+Xlqkaq3EStoiiPYL0krMqqNyFXs82cTa2cPoxxbjE9vI6+dt3f6URWZqbxqI9VfiGOb8jzdijJH9Tby50fk1qx+1UZ5XrtPryJVVWRW2j0GDyPNj6pIUSd5/Xe8+vckIp8qpCrzHQrkWH6PQ9UPFD+gEiB5CZywSV4AAACAmgQKdmtBzcszzzxDJpOJxo4dK9sKCgpozJgxlJiYSFFRUXTVVVfRoUPG+UcAAAAAUH5mzJhBaWlp5HQ6qWfPnrR69Wq/27799tt07rnnUnx8vL4MGDDglO1vuukm/fNbXQYNGkS1PnlZs2YNvfnmm9SxY0dD+7hx42jhwoU0f/58Wr58OR04cICuvPLKoF0nAAAAUNVqo8osFWXevHk0fvx4mjRpEq1fv546depEAwcOpMOHD/vcftmyZXTdddfRDz/8QCtXrqSGDRvShRdeSPv3G0sLRLJy8OBBufz73zyRaq1MXnJycmjYsGF6dieyuhIyMzPp3XffpZdeeon69etH3bp1o9mzZ9OKFSvol194llwAAAAgFDmZgJgqsVCFEZ+pt956K40cOZLatm1LM2fOpIiICJo1a5bP7T/88EO68847qXPnztS6dWt65513yOv10pIlSwzbORwOSk1NlYv6eV4rkxcxLDR48GC9K0pl3bp15Ha7De3iwTVq1EjP/vxRWFhIWVlZhgUAAACorWSV+swTn4O+KCoq0j9b1c9Vs9msvy7rc1UlLy9P/2xOSGC7mpIemjp16lCrVq3ojjvuoGPHjlGtLdidO3eu3m0lho1Kk56eTna7neLi4gztKSkp+jp/TJ06laZMmXJK+4URxygmwkyTH2TlysQ2/5HxvP8oP8xjnPCkfWTM7x6yXSXjzf94R8Yv9+cUOOI/R/geN3SX8ZMXbpCxt3VjGSf8yt5EadZoGR9razx31F/8ulBjhZFZ8QuK3sUFXFm9832qdOLNvH0uWxbRxnyWJ50X9afh3E7lfZivKb8Yubky3J2XyKezsJeSTckfvS7FQyqXVUXuCPZY0tyKukjsr6iNTBZW2sTZ+P7221npFG1SvY0U7yAlV1e9jVRKq41UDyNSPIzcSruqKvIavI0UPyKy+PQ2Un2K7Ip/kSBTc/lUIqmqIotJUSF5LadVFZn9KBVKf4vx72Fkqn41RCUUPKpyzP/2gVzTGSiMDOPiy3ClqtRGDRs2NLSLIaHJkyefsv3Ro0fJ4/Hon6Mq4vWffxr/5vvjwQcfpHr16hkSIDFkJEo6mjRpQjt27KCHH36YLrroIj0hsih/s2tF8rJ371669957afHixXrRUFUxceJEfTyvBJGFlv7BAgAAAMFGq6R6XlM+T2NiYgxDONUlrBGdDqKXRf3cHjp0qIw7dOig1682a9ZM365///61a9hIdF2JAqGuXbuS1WrVF1GUO336dD0WmaDo4spQ3JcFQm0kxtP8IX5o4oeoLgAAAEBtJabUZ56/5CUpKUnvCSmt2j3d56rghRde0JOX77777hRxTWmaNm2qn2v79u1UXQQteRHZ2B9//EEbNmyQS/fu3fXi3ZLYZrMZioK2bt1Ke/bsoV69egXrsgEAAIAqoXLFuqYKDzmJUgwhflE/V0uKb8v6XH3uuefoiSeeoEWLFumfzadj3759es1L3bpKXUJtGTaKjo6m9u3bG9oiIyP1OV1K2keNGqUPAYnCIJFN3n333foDPvvss4N01QAAAEANGzeqAOIzdcSIEXoS0qNHD5o2bRrl5ubq6iPB8OHDqX79+nr9qODZZ5+lxx57jD766CN9bpiSmlMx95pYhGJY1JmKedhE742oeZkwYQI1b95cl2CH5Qy7L7/8sl4JLR6KqJ4WD+L1118P6FgXbLiOLBEOWnkWy8GizDxm9/htPK1+1LZYGdebZiwmTk7oJuP9fbhg9Jpeq2T8R5pSjLuCu+8yB3Ah6eFuPL1/yuxffRbDWtoZlVLx73NR619KUas5mQtlY//mQt7kf3LhsCUm2mdRY0EqF4JuzuYseXg834/AeZx/S457udDWm8v2AHuU/WPs3G7P5uN4orgi1r6Pi32L+dZI8xinyXc5i3wW7MZYcngjG7+VI828vcehTsuv2ADwzPsGvIoFgL6/xgWxZhvHXuWvht3KP4si5dnazaoNgMVP8a1aTGs8t1dZV57CXK8yXb/xOH7a/WwvUCWYlnLYABiLf3136BrtBCgAzkTRbPWfApQPxeGi9lLJgl0KYN9rr72Wjhw5oickIhEREmjRo1JSxCtGN8TnbglvvPGGXsLxz3/+02dRsBiG+v333+n999/XyzxEMa+YB0b01FRX7U2NS15EcY+KKAgSMwGKBQAAAACV56677tKX8nwO79q1q8xjuVwu+vZb9vk7U9So5AUAAAAIFwKdJfeMmo/WUJC8AAAAAEEArtIhPMMuAAAAAEBFQM8LAAAAEAxEz8kZLtitLYRN8pI0zUFWq5N+eI/9GH7ObiHj9y94W8azOp0r48MfspJHEP/dNhnfdyc7XL+R9pmMLzmf9fJ1Vh6X8b+zWsr4RFdFFfQ6q5bWFnJ19qVNNxrO/fs23v/bHJaZFzfgqfFduzNlfFYiF1qtTW7pU9HkSmXFz/bjfK8paca3hvMYK2oOFLNKSyvi+ziWxYqtWBdvY8/igVl3NB/Xns/qq2LelUhR+AiinXy9JkVVlGBltZHmZBWTU1Xz2H2rYDylbABK8NpLKX4U+YnFqtgDKIPNNoOqyOzTBqBI4+u2+bEHiDAb/UgMCiVS7klR3ZgVSUZ5VEgWk28VksVkKrcSydc+WkXH5LUq7Ar3t30Y1wOA0AA1L4GDYSMAAAAAhBRh0/MCAAAAhPskdbUFJC8AAABAEIDaKHAwbAQAAACAkAI9LwAAAECwCOOhn8oQNsmL6ZdNZDLZaMIHI2SbXbEOmnLfWhl3abBYxuf+c5zhOCmvs+fP1q97yjjpbjbnOd6fVTSJH/4t49c295Vx3/ZbZXy4LluRf3KcTXeGJ/1sOPfv+9if6OuD7WSc39Ql4/gvWWHUK4qVUSvqnSXjQx72/mmZzP5Hv21v6NP3SeA4zkqYXW5WN2nFrDYqzOB9TC6+JkcWK2WKorizL7KAj+mJMCqMVOIcrMby2FkmFGth/ySvg9sjFB8gj0NVG5lO622kWUurjfhYVquq+OHtnIq3kaoQUtVGarvT5Lvdrqik9HWqesivt5Hit6T6Cym9ycVec4V8ispa588nqTyeR/6wKJ5TZ+KPfKh55lT4ekPs/sIZDBsFTtgkLwAAAECNAgW7AYOaFwAAAACEFOh5AQAAAIKCGPapzNCPicIVJC8AAABAMMCwUcBg2AgAAAAAtb/nJTc3l5555hlasmQJHT58mLxeo1Lk779ZYVNTyL66O1nsTmoyfYtsMykqh+uvuFjGLzb+VMZpV+8wHCd/Nat8Gn3JvkX/GRkv4/Fdv5fxF15WEllXslroljHLZfxY+1tlvGhHioxfqLvScG5PdraMd+1oI2NXGncdxmSyhKqt7ZiMcxuwEmibm6+1W9weGW863Iz8YclgZc/2Qr5GFWsGq2AogtVGtixW1+SkKr5Ihaw20iKNShuVBAef+6iNvZ9iLKzq8jpVNY/Jp9pIVbX4VRvZje9lj+phpKiN3Mo3HqdF9TBS1EZKu1v1NjKdvv3kOt9eRap/ktpe7Kfdn0LIo6iQVCVW6X3UdYF4FVVYJVETv01W9L5r4j2Amgd6Xs5s8nLLLbfQ8uXL6cYbb6S6deuSqZSpGwAAAABOA1ylz2zy8s0339BXX31F55xzTuBnBgAAAAA4U8lLfHw8JSQkBLIrAAAAAP43DOt3KLac+4crARXsPvHEE/TYY49RXh7XIgAAAAAggJqXyixhSkA9Ly+++CLt2LGDUlJSKC0tjWw2nppdsH79+qq6PgAAAACAyicvQ4YMoVCj/R1/kD3KTnuXxck2LSdXxvveaSnjG68fLuMF7T4wHOecS++XceNHVsj48U2DZby6x2wZf9mql4zrruDz9RjPqpbDXVn6YtmoxOcaO8ZMVk4So//iH112O/YXIo2PW9fCip/sBlzY9Vt+Yxl3imC1keswH6ZQU44pyMqR4Z85rKAyWbjdkaF4B0Wzqsiazaoid6SiNipijyVLBJ/PZLEY1UZ2fm5HHazYijaz55HXwfvYFFWR0dtIabf79i8y2Up7G/Frh8HDiI9rNysqJFLURia3TxWSTfEwyvM6fLafPIe6T7FPbyOzYnxjVAj5UxUp91amt5G/dlPwCgqVa1J/lv4VUAGcO4wLIEEQQMHumU1eJk2aFPgZAQAAAKCbblbGKNSEYaPAWLduHW3ZcnLelHbt2lGXLl2q6roAAACA2g3meTmzyYuYmG7o0KG0bNkyios7OQyTkZFB559/Ps2dO5eSk5MDvyIAAAAAgKpWG919992UnZ1NmzZtouPHj+vLxo0bKSsri+65555ADgkAAACEZ81LZZYwJaCel0WLFtH3339PbdrwFPVt27alGTNm0IUXXkg1kWn1V1NMtIWa33e7bIvZwT/4lHdYIXXMzMNfnseN/XL9BvJ2u9+oJ2Pr0lje5yze50ivJBknz/3dZyGouxsXvdb5NxfZ/uXmglSBJYV7tOK3cQFn4kVcaWuJ5oJWm4l/vHkNuCh1XSYX7F4e/ZuMXUf5mk54uchWoGXzNe7M5uuItHPRrT2Dty+O5kJU+4FMbo/i5+R18z1EuIp8Fibr92fj/cnBx4028zV6FHsAm1qYy5sb8Nr5Xj1KkbNZsQAQuJViXruyrkg5h9Pi9llk6zD7swHw+LYAMBmtCdTCXMN0/8p0/QZ7AKUw13Cvfpxn1UJXS6lt/BXmqu3GotmqtBOoxX+Qw/jDBvgAw0ZntudFeBmVlkcLRFtpnyMAAAAAgKAnL/369aN7772XDhw4INv2799P48aNo/79+1fl9QEAAAC1E0xSd2aTl9dee02vbxET1DVr1kxfmjRpore9+uqrgV8NAAAAEC4geTmzNS8NGzbUZ9EVdS9//vmn3ibqXwYMGBD4lQAAAAAAVOc8LyaTiS644AJ9AQAAAEAFwQy71Z+8TJ8+nUaPHk1Op1OPy6ImyqUfPNiN7Nk2+vcVPKw1+2gfGe9ewC7ZSZ+dnHhPMHLE1YbjzGn+iYwHXcRWAXWXHpXxu7e3lvGx3qyiSXiHFTsrC3l++qtb/irj9Vt438+zOxnO7W6aIuPIbSdk3DNlm4xXpbICLF9jNY6rQbaMtxytw9fdmN8CrsOsjtlbbJTpePNZ+XToRIyMm7lYYuTMUBRUMXxc+3be1x2lHFRR+cRG8DYmm/FtmWDl56Y5uVA8Upl+v1ixAbCZWKXjObWu/BS1kar8stpLT9HP65xWVVVk9qkqKlJVRQa1EV9ThKKSMlgAUOlzK9P6K1Np+lMhFSvbW0y+bQAsJsXCoQxVj+ZnH7/ioQqqiipsM6Dv5Gefinadh3FXe00knGeJxQy7ZyB5efnll2nYsGF68iLisnpkamLyAgAAAIAwK9jduXMnJSYmytjf8vfff1fn9QIAAAC1gyAV7M6YMUMX3IjOiJ49e9Lq1avL3H7+/PnUunVrffsOHTrQ119/bbwNTaPHHnuM6tatSy6XS69/3baNRwRqjNro8ccfp7y8vFPa8/Pz9XUAAAAAqHnMmzePxo8frxssC+FNp06daODAgbrtjy9WrFhB1113HY0aNYp+/fVXGjJkiL6IWfVLeO655/RykpkzZ9KqVasoMjJSP2ZBQUHNSl6mTJlCOTlch1CCSGjEOgAAAACUjUmpewlooYrz0ksv0a233kojR47UZ8YXCUdERATNmjXL5/avvPIKDRo0iB544AFdVfzEE09Q165d9SlTSnpdpk2bRo888ghdfvnl1LFjR5ozZ44+D9yCBQuoRiUv4mJFbUtpfvvtN0pI4MJXAAAAAFQvWVlZhqWw0GjvUkJRURGtW7fOMK2J2WzWX69cudLnPqK99DQoolelZHtRLpKenm7YJjY2Vh+O8nfMMy6Vjo+P15MWsbRs2dKQwHg8Hr035vbb2TuoJrF2Zmey2J008akfZNtr9fnBth81RsaNn2f1T/rHbQ3HcTzMSg/3paz48by7XcavrO0n4392YS+kTWnsKfTuoZNu3IL/q8fjh2t3sxznk93ssSQwtXDKOGkDn+/8KFZHLW/cW8b7ilkd0ymVZ0Ne+UcLGUeZ+ZjOI6z4+bOoruHcmkfx4znG+5giImTsyORtCuIUdY2iVCqOMipqSkh08jBkkeJfJEiw5srY62L5UITiBVTsMvtWG/GlGu/HpqqN+DjWUt5GRarayOJbPeSyFPlsd5p4+wKNrzvOxPfq9vr3NvIq6iG7okRSVUVmf6oi5TuZx4/nUVmKH4/39N5G5T2WvCZTOb8rVZGCItSUGBW+3hC7P1B9UumGDRsamsWQ0OTJk0/Z/OjRo/pndUoKK1cF4nXJnG2lEYmJr+1Fe8n6kjZ/2wQ9eRFdQ6LX5eabb9aHh0R2VYLdbtcLgHr16lUd1wkAAADULqrImHHv3r0UE8NTWDhKfQGsjVQoeRkxYoT+r7AC6N27t09zRgAAAACcOWJiYgzJiz+SkpLIYrHQoUOHDO3idWpqqs99RHtZ25f8K9qE2kjdpnPnzhT0mhcxjlZCly5ddGVR6XG2kgUAAAAANUsqbbfbqVu3brRkyRLZ5vV69df+Rk1Eu7q9YPHixXJ70ZkhEhh1G5EHCNVRdY7EWCtS73Lw4EGqU6cOxcXF+SzYLSnkFWNqAAAAAKhZM+yOHz9eH0Xp3r079ejRQy8Hyc3N1dVHguHDh1P9+vVp6tSp+ut7772X/vGPf9CLL75IgwcPprlz59LatWvprbfeOnkNJhONHTuWnnzySWrRooWezDz66KNUr149XVId9ORl6dKlUkn0ww9c9AoAAACA0ODaa6+lI0eO6JPKiYJaMbSzaNEiWXC7Z88eXYFUgigR+eijj3Qp9MMPP6wnKEIC3b59e7nNhAkT9ARIWAhlZGRQnz599GOKSe2CnryIzMtXHCrEzF1LVpONevdn64Lne8+X8W3XfiPjTzZeKOO6/2FVj+Dp0T1k/GqHuTKeGnOejJOXsm/Rnef/KONhPdkLacvvXJnduvFynx5CGZtOzmhcgqkl93bFZ7NXUVs7q3Gy0vjcGwrry/icOL6P9ensf6RiPsFz92zK431LYz/OChktNpLbT7DqJqsRq5C0fGWioihW4JCiPkl28rn3O4xjt3EWvj+Pk9+yTlXtptSnmZXRUC8/DiMOVvZ4FEWR3VbKX0hR7TitqoeRxae3UYHGJ3SYWe2Vp1yITVEhFSoqpPJ6GxlVRarn0elVRWaDCsl3e+l9jCt8N/tTTFT4OGXg1z+pKqmgRxMUP6AmFOxWlLvuuktffLFs2bJT2q6++mp98YfofRET1J7JSWoDmudFZFQ//fSTYaphkb1df/31dOIEy4dPxxtvvKFPaFNSbCTGx775hpMIMTvfmDFjdFuCqKgouuqqq04pHAIAAABCkiDZA9QGAkpexEx7JYW5f/zxhz6GdvHFF+uT1Yi4vDRo0ICeeeYZfdIcMYbWr18/fYa+TZs26evHjRtHCxcu1H0Vli9frs/Yd+WVVwZyyQAAAAAIR6l0CSJJEdMKC/7zn//QpZdeSk8//bTukyCSmPIi9lN56qmn9N6YX375RU9s3n33XX2sTSQ1gtmzZ+vTE4v1Z599diCXDgAAAIRtwW5Y97wIuVWJMeP3339PF154skZEFPQGKpUWCiVRxSyKfsTwkeiNcbvdhimHhatlo0aNypxyWEyLDOk2AACAkJlhtzJLmBJQz4uoJBbDQ+ecc45upS1cKgV//fWX3mNSEcSwk0hWRH2LqGv57LPP9F6dDRs26EmSkGVXZMphIe/yaQ55Vlsiq5NaP8+FoQ+OHCbj7dfNlPHM63n69sgvjDU8Hy/h6fefvu53GU86u6WMk5btk3GaNVrG6bwrJaznR39iMJ/PosxanHBy9EyScxknYmY7F4AmmrloNjuNt1+R3VzG1yf8IuOIg7xNlpcLhLWMTBlvyTJOWGSyHpOxg0PyxLj42rP4WEXRSsFuERfyOqI4Nlm46DXJrhTsOpMN544z8/PxuNSCXd6/2On7l1gt5FVtAExKYa5baVeLcvV1yh8Hp8Xtc7p/tWBXtQewK4W5mRo/J5tJObdSZFvaHqBYtQ7wU5hr9luYSz4LfMs7pX9FbQD8FtMG8s1Q81187X/7Cv4BPxN/8MP4QwXU/ILdsO15EW6SVquVPvnkE32YR2jCBaLYVrhPVoRWrVrpiYqY0OaOO+7Q9eebN2+mQJk4cSJlZmbKRUybDAAAAIAw73kRQzdffvnlKe0vv/xyhY8leleaNz/ZQyBm/luzZo1uwS206MIBU2jG1d6XsqYxLvF0CAdfBwAAAKENal7OcPJSUqMiJqrZsuWko3G7du3osssu030TKoOYqljUrYhERngniSmHhURasHXrVn0CHZg/AgAACHkwbHRmk5ft27frqqL9+/frwz4ltSbClvurr76iZs2alXuI56KLLtJ7crKzs3VlkZgg59tvv9Udq0eNGqXX1ohCYDEPzN13360nLlAaAQAAAOFLQMnLPffcoycoQrJcYhlw7NgxuuGGG/R1IoEpD4cPH9Z9FIRnkkhWxIR1InG54IIL5DCUmKZY9LyI3piBAwfS66+/HsglAwAAADWLSg4bEXpeKoaYME5NXARiFlwx4ZxQIJUXMY9LWQhfBDF7r1gqy8GxHrJEFFODa3fKtlZvsepj6oWsFvqg+ywZ3z9wjOE4Tb4olPHiIfz49g7guOm3XCS8zc0qmrPP2irjY/9iVdbXuRxrTXla/oSNbAEgOOvubTLeWYcVOW6NVS3FTVjxs/pIYxk/ksI2BVEH+b73evjdr+XwNPw7jtUznLtRJB/XeYL3KYpl1VPEfpYhuWPY2kBTjDpjI9gqwGRnxU6K7Shv7zLO6R9tZoVSsYtrzB0mq09VkYrXwdfq0VjNY7Erih+l3WllRdHJdXw+l6I2chMPj0ZY+D1R4OV7ijBze6HSrqqQVHWSrbTaSDm3QW2ktiuiFmO7YgNgsBMw+VQOqdvr68i34kdVFZW2FKgYZ0LxU/2nAOUjnGszygTDRmdWbSQKYsUwT2lycnL0AlwAAAAAgBqVvFxyySW6e6SQN2uapi+iJ+b222/Xi3YBAAAAcBrgbXRmk5fp06fr8mZhlS2GdsQihotEm5A5AwAAAKB8UunKLOGKtaIy5ueff56++OILfQ6WIUOG6JPKCTts4TlUMl8LAAAAAECNSF6EceLkyZN1vyGXy0Vff/21rhKaNYsLXAEAAAAAakzyMmfOHF2qfNttt0lTxsGDB9M777yjS5prMt93/RfFRJvp3NHjZFvKG6tk/OFH/WX84N1/ynjvdUb1SYub2XBo7IZrZdznXG4/XJdnAH7tyPm8fd3FMp60ldvf3dNHxvnt2Nso/suTEwCWcEn8Bhm/2PR6Ge/3sBKoQ8MDMv5te0MZJ3Zk/yNXOm+/ubCujL2KB1HuUd5eYIpS9j/GSp3CeEV1k8vHdccYlTMlJEew+srjZIlQkpV9m7wuo3QoWvECKnap/j2Kosbp83TktfN1FBMfx2bwNuK+V5fN+PMu0PhXxGVWvI0U9ZDTj3rIbvAwUlVFHj/+Rf69jcyqqkj1Q/KjHlLx1+7P8yiQY6lj7xaTuULbn7quYvuEUtd5QNcaQvcHKgjURgFToYxDzG4rJqcrQfTAiCGjAwf4AxMAAAAApwc1L2coeSkuLtaLc1XEFP5ut/HbKgAAAABAjRg2EpLom266yWB8WFBQoEukIyN5WOHTTz+t2qsEAAAAaiNh3HtyxpIXoSwqjbAEAAAAAEAFQc3LmUleZs+eHfiZAAAAAACC5W0Uivy3IJ4ibBYadvt3su2Lfawwavze3zKeMrSDjN/p/b7hOM852LspamG0jB95/BsZj+hzH5/jV1b8vDJ4jYw9ir3C3g3tZWxqx0qLmA8yDOc+y3FCxhktXDJeV8g+RP2SWCn15w++3b0tR1jZ82se+x+pOA4b3xpaTBSvO8aqpOwGETL25uUp27MChxT1SR0X33e6k59folVRIUUYz+1UfHdUtZFN9TbyozbSHKzg8SjGPA47X1+Bom6JKOVtVKSoh1wWvu8CjW0wHIoKKc/L7TZ/HkaK6kn1Tiq3t5GiNlLxKO0GJZbXt0LLq7SX+xtdJVRINYYylU4B7ANAgFS26NYUxu/LsEleAAAAgBoFho0CpmZPzgIAAAAAUAr0vAAAAABBAMNGgYPkBQAAAAgGGDYKGAwbAQAAACCkCJuel0fnXU8Wp5M2jZ4h2z4Z3UXG3u9YgTPv8/NkPOUW9iwSPNa/rYyTvt0p42ZPsxrnwABWjSSv4Ed8eBAraiwJ8bzNek6f869jRZHZYZTQJJp5IsDMlrzPkgy+ptHJy2X8/r7BMj7hVZRAx1nF9OuJBnw++xEZO48aTk2eeFYVWU7wsQpjuV1TvJFc0QV8XBs/g3qOTBmnu5KVe8uVcXEptVGEoipS1UaG63Py8/AqHkFmh+ph5PXpYeRW1UaKokhQoNl8rjN6G/GxTnj5Z+RUVEiFXqtPz6MipV1VFOn3alAP+Vb2qO2qqqiiPkXmUt9jjOtU/6SKfgM8vQrp1HP7O0cZ6qiq2D4QzsQ5QO0FPS8BEzbJCwAAAFCTQM1L4CB5AQAAAIIBel4CBjUvAAAAAAgp0PMCAAAABAP0vARM2CQvaTO3ktVsp8v7XizbvurIU/8Puv5+GTedx9Wqr1yVZjjOvsu42LLFgnQZLyvgwr1hPVfKeP3LrWU8O7OzjIvb8rT88b8e4+uYyAXCqxryvoJ8rVDG9hZcYLziQBMZP1eXC3aj93LB6HY3F5h6FWuCnYf5OppFcSFuxGHjb0VhIk97H7n7sIyL4pJkrHn42SRGcwGuyc771rUf5O0j2J082szFsMURxg5Bm4mn1i/2YwOgFuy6Nb4Om4On6C/UuGA3wqZO9a9aALj9FuyqNgDqdP8R5sLT2gCoBbtmZaC62GvxW7BbpK4z+bENUKwTvEpxrMVQZGvyuX2Zs+RXsDDXrz2A3xMEYE1QVduDaiOcazACATUvgYNhIwAAAACEFEheAAAAgGAOG1VmqSaOHz9Ow4YNo5iYGIqLi6NRo0ZRTk5Omdvffffd1KpVK3K5XNSoUSO65557KDOTp8cQmEymU5a5c+dW+PrCZtgIAAAAqEnU5GGjYcOG0cGDB2nx4sXkdrtp5MiRNHr0aProo498bn/gwAF9eeGFF6ht27a0e/duuv322/W2Tz75xLDt7NmzadCgQfK1SI4qCpIXAAAAAEi2bNlCixYtojVr1lD37t31tldffZUuvvhiPTmpV68elaZ9+/b0n//8R75u1qwZPfXUU3TDDTdQcXExWa1WQ7KSmppKlQHDRgAAAEAIDxtlZWUZlsJCFhEEwsqVK/UEoyRxEQwYMIDMZjOtWrWq3McRQ0Zi2ElNXARjxoyhpKQk6tGjB82aNYs0vwoB/4RNz4vJ5SST2UHZL/B0+Mdn8ANrefMWGR99h6fof+0b7toSPHLJZzKe36WfjB/d1lTGC9p9IONh21hp8+7G3jKO6MbT6td9Y7OMr4hZL+MlbfsYzr3ZzQqN/o22yfjLn7vKOOosluO49rOqaE1+E5+qoOJ0l4xNMTG87xGj6ianvjJNfo4ylX8cq3lU6kbwuXNcfE0pNh7/9ESqaiNWArkjy1Ab8WMzoDm8Pu0BHHa+viLlFyTCyvdXoPGvQaS10L89gKKIUtsTzDk+bQNUG4BiRYVkV1RIqnLIXEqAo9oDqOohj8E2oDztFbMNKGudUblUju8+VditXWVd5GdAoRHQtYaxciRsqSKpdMOGDQ3NkyZNosmTJwd82PT0dKpTp46hTSQgCQkJ+rrycPToUXriiSf0oSaVxx9/nPr160cRERH03Xff0Z133qnX0oj6mIoQNskLAAAAUBvZu3ev3sNRgsPBXwxVHnroIXr22WdPO2RUWUTvz+DBg/Xal9JJ1KOPPirjLl26UG5uLj3//PNIXgAAAIBQQPRjVsba0/S/f0XioiYv/rjvvvvopptuKnObpk2b6vUohw/zfF4CUbciFEWnq1XJzs7Wi3Gjo6Pps88+I5uNe6N90bNnT72HRgx1+Uu6fIHkBQAAAAiDGXaTk5P15XT06tWLMjIyaN26ddStWze9benSpeT1evVko6wel4EDB+pJyBdffEFOp59ZRRU2bNhA8fHxFUpcBEheAAAAgCBQU6XSbdq00XtPbr31Vpo5c6Yulb7rrrto6NChUmm0f/9+6t+/P82ZM0cvvBWJy4UXXkh5eXn0wQcfyOJhgUiYLBYLLVy4kA4dOkRnn322ntgIGfbTTz9N99/PM9yXFyQvAAAAADDw4Ycf6gmLSFCEyuiqq66i6dOny/Uiodm6dauerAjWr18vlUjNmzc3HGvnzp2UlpamDyHNmDGDxo0bpyuMxHYvvfSSniRVlLBJXrbd3YjMTic1G8++QxcPGyPjzee9K+PLug6XcfOP2ENIcNP17M3zwiU8sY75B94mpj13f5msPN4XuSJSxpldWdWS4mZFTGtlfPBYe+OP55usjjK+JP5XGS/b2d2n/5HpEHsm/ZzBbyaThe/JdZAVI97EaBnbj7LPkaCgPd+rN5fXWeOU81lYUdMgghVbf0bwnAB1LKxCKo5SlDwGRZFxFNisKPo9Si+kqioiJyt4ChRvI6eNVUWFilImylZ4WkVRafWQ08THyvRE+GxXPYxsJsVXyePb26jIo3ohGb9GGdVDvttV/KkN/SqH/KiQTq6sqFfR6dvVn2OZysiq8kmC4gfUdGqwMWNCQoLfCekEIhlRJc59+/Y9reRZ9Oaok9NVhrBJXgAAAIAaBxLmgMAkdQAAAAAIKdDzAgAAAASBmlqwGwogeQEAAACCQQ2ueanpYNgIAAAAACFF2PS8zL50JkVFm+n+xXfKtrQ3WZWyrAfLWLYP5ZkKm05gdZJgh5uVNl0u4mmUj41lz6T/3KxMAtSWPY9Sf2aVT68bt8t4d/26Pq+5oF2+4fU3B9rK+I7262Qcu4vv4+9ijr0Z7CP0W3oLGTeKYpVOZDqn7gXJ7HMUsZlVVYLCRFYbacWsrkmI4edhUiYZauTg/bdENePtLby9O4qVNhEmVvW4/fgXCTwRfL0eje/D6mRlj1tpj7SrfkR8vkirb5+iKGuB4Xx5Xr6nCDMrlA66+XnYVVWRQW3k9elhZCff7ZbS3kaGdYpXkeovpMzP6fX63t6rbO9P8aN6IZVeZ1zhp71S84SGwLfMiiqgajnhPFxRlWDYKHDCJnkBAAAAahQYNgoYDBsBAAAAIKRAzwsAAAAQBDBsFDhIXgAAAIBggGGjgAmb5CXFUkjRFjM5JxyQbd7z98v4tm9ulvFdg7+V8fezeep9wT1/15fxO83nyfiWDf1k/Pjvg2Xs7MVT7qfM5in9RyX/V8b3d+Ai4j/dXAzbr8VfhnN/v7q9jBM7stVA5E6ecn9FHhfHeou4KDV/H1+HKY4LkiMPcrFpXgq/HVwr+ZiCogQuBFZpGJPB+0dypW09G9sDeKO4GDrOzMdxR/GopU21B+BbOwWvi/cvJo4dDr6PAqXaNNrORba5SmFupEVpNxTllrIHUPZJMOf4sQ3gcxcpBbt25frUdrNS+1msFtmWKnr1ZwPgv91UMXuAsopQlT+KFpNa5FvBqfgDmLo/lL5NVvhaQ+jewBkAyUvAoOYFAAAAACFFUJOXqVOn0llnnUXR0dFUp04dGjJkiO5SqVJQUEBjxoyhxMREioqK0p0thaU2AAAAUBtqXiqzhCtBTV6WL1+uJya//PILLV68WLfYvvDCCyk3N1duI6yzFy5cSPPnz9e3P3DgAF155ZXBvGwAAACg6oaNKrOEKUGteVm0aJHh9Xvvvaf3wKxbt47OO+88yszMpHfffVe35e7X72RNyezZs6lNmzZ6wnP22WcH6coBAAAAECxqVM2LSFYECQkJ+r8iiRG9MQMGDJDbtG7dmho1akQrVxpnvi2hsLCQsrKyDAsAAABQ0zBpWqWXcKXGqI28Xi+NHTuWzjnnHGrf/qSqJj09nex2O8XF8VTsgpSUFH2dvzqaKVOmnNI+6Mc7yOxy0vYL3pFtF/ccIePWb/JU+uOv3Cnjt4cOMhzH9lWSjJPGsYqGLKyWcS5lZU9mL55uPvkNVrh0sfOjP9yNlSufZLK66drEVYZzr97WScb5Gh/LfPCIjJcebyNjk4WVQBH7OE/1JPPzdKTzEN2xdtzuzeVp/AW2RL4Pk3KvTSKPyXhTBCuxUq38PN0xdhlHm/i+i6J8T1tfXMoewKtMp09OVvAUaBxHOlglVKgoXKJshT6VQzGKDYBqARCpWAAIjhTzz9JpcvuxAVDsATy+7QGKPPzMbEpfr1tpN5ehKlKn7/cqqiK13a+qyI8Kqawp7/3v428HOq0FQXnP7f+iAr+mcrWXdQ4AqgOojUK/50XUvmzcuJHmzp1bqeNMnDhR78EpWfbu3Vtl1wgAAACA4FMjel7uuusu+vLLL+nHH3+kBg3Y4DA1NZWKioooIyPD0Psi1EZinS8cDoe+AAAAADUZzLAboj0vmqbpictnn31GS5cupSZNmhjWd+vWjWw2Gy1ZskS2CSn1nj17qFevXkG4YgAAAKCKgNooNHtexFCRUBJ9/vnn+lwvJXUssbGx5HK59H9HjRpF48eP14t4Y2Ji6O6779YTFyiNAAAAgPAkqMnLG2+8of/bt29fQ7uQQ9900016/PLLL5PZbNYnpxNKooEDB9Lrr78elOsFAAAAqgoMG4Vo8iKGjU6H0+mkGTNm6EtlaDkth6wWN73avals++s2VsG0uPl3GS8rYMXBPy9nDyLB+qGtZTztJlb2eLq1knHdpaz+GXL7Jhmvasb7Fmq/yNjUlZU5n+/qIOMHuq0znDt+K6tdNhYp6qFjx/n69rH/UbNYVrtE7ednXVDXJePI9ftknF+H64q0Yj6XICWeJedmF++f5tjN1xTTUsbJFlYxFUWzosahqI3cfjyMiiMVdZHYTlEV2SMUxY/G20U7WD2Uq/E5oq1+PIwUbyPVpyiilNqo0OBhxOfO99h8q4oUFZJF6dMt8vIzsKjeRpribWQq5W2kKF8sflRF6j5epd2f4segTjI+5nL6HlVQuRTIH9cw/oNc0wjnD8czAtRGoV2wCwAAAIQb6HmpBVJpAAAAAIDygJ4XAAAAIBhg2ChgkLwAAAAAQSKch34qA4aNAAAAABBShE3Pi7ZjD2kmG7335kWybd646TKecNEdMr5lZRcZb+nLXkiCS/5k1ciby046XQuc/bm94eOsXBoVv1rGX/f6h4yXF7BnzrAWa2X83sL+Mo46S/FOEkqYHawq+ia7I9+bh9U42k6W8JiSE/lYe1mNc6I1q4VcJ9j/qLgO+wOVplkMexgdjo6ScWP7Ud4/lq83wcxfJwpjOUe2mfg5ufkRGPBG8P2UVhu5nHyNBYqMJsauqopYRRZj8+1hFG323Z5sNRp55nnsp/U2spPHp6rIpnylKlZ8ilTlULHB28hULm8jjx/fIX9+RJpyHEO7QbVkrrwvUHV/26xKD6Pqvl58mwblQfwNq4y5oha+b7SwSV4AAACAmgTURoGDYSMAAAAAhBToeQEAAACCAdRGAYPkBQAAAAgCYnJuZYLugPYPVzBsBAAAAICQImx6Xg6N7EIWh5NS31wv29pMYJXI8dvYi6fBO6ym+f0cY2prbdxQxmkLeZ1r4l4Za6/EyLi+hSU1h3vx9m8fYOXRi40/lfEXm1jBtKc423Bu7/6DMl60v62MY137ZRy9i7cvrBcrY8feEzLO68tqI28+q25ik3JkbHYYlU4tIw/xfcSw/1J9K6uViuJYmRNtZu+fohjfnjvFUdzn6daKZWyJ4Fi/D0XNE+1UVEWaxbeqSFPURpZ8GWd7+Z6iLbz97sIkGTvNRk+nfEVtZDepqiKrb28jg3qIcXt9t6vKIVWFpK/z+P5u4TWokFQPI/U5q15IgXQ5n96ryJ9/UqU9j/z6KlUR1X38ECScCz+DCoaNAgY9LwAAAEAQ1UaVWaqL48eP07BhwygmJobi4uJo1KhRlJPDX3B90bdvXzKZTIbl9ttvN2yzZ88eGjx4MEVERFCdOnXogQceoOJi4xfW8hA2PS8AAABAjaIGz/MybNgwOnjwIC1evJjcbjeNHDmSRo8eTR999FGZ+9166630+OOPy9ciSSnB4/HoiUtqaiqtWLFCP/7w4cPJZrPR008/XaHrQ/ICAAAAAMmWLVto0aJFtGbNGurevbve9uqrr9LFF19ML7zwAtWrV4/8IZIVkZz44rvvvqPNmzfT999/TykpKdS5c2d64okn6MEHH6TJkyeT3c7D9KcDw0YAAABACA8bZWVlGZbCQq4NDISVK1fqQ0UliYtgwIABZDabadWqVWXu++GHH1JSUhK1b9+eJk6cSHl5eYbjdujQQU9cShg4cKB+zZs2barQNYZNz8vQUd+TM8pK33/PP4yrt10p46+7vSXjW67hotnh60YajuMYzMW4KbN/lfHkmf+V8YRebDXwR9EiGQ/ovlHG369uL+O0ZlzUG7cxU8bf5rY0nNurvAkObeMi0/g6/EaN3ckFpzkNlWLT9Ty9f0FqMh9U42LT5gm8TV4MFy3r6xx/ynh5XE8ZJ1uUKfPjuCjVYbKd1gbAE8UFsMVKUa4rwmhTkOvla4x3qAW4fH9xNn42WV4uSI61cnuuYgNQz6YUMCvHiTQZz53v5ftwmnhctsBj82kDoNoD2E1cGOpWCnktaruyfWkMxbzKPl4/BadeP/YAalGfagOgFviWtU9lCnBDrRAUNgDgjFJFBbsNG7KQRDBp0iS9JyNQ0tPT9XoUFavVSgkJCfo6f1x//fXUuHFjvWfm999/13tUtm7dSp9+elKUIvZVExdByeuyjhvWyQsAAABQG9m7d69eWFuCw8Ff1FQeeughevbZZ087ZBQooiamBNHDUrduXerfvz/t2LGDmjVrRlUJkhcAAAAghL2NYmJiDMmLP+677z666aabytymadOmes3K4cOHDe1CESQUSP7qWXzRs+fJXvrt27fryYvYd/VqNisWHDp0chqOihxXgOQFAAAACAO1UXJysr6cjl69elFGRgatW7eOunXrprctXbqUvF6vTEjKw4YNG/R/RQ9MyXGfeuopPTEqGZYSaiaReLVty3OXlQcU7AIAAABA0qZNGxo0aJAuexY9JT///DPdddddNHToUKk02r9/P7Vu3Vr2pIihIaEcEgnPrl276IsvvtBl0Oeddx517NhR3+bCCy/Uk5Qbb7yRfvvtN/r222/pkUceoTFjxvgd6vIHkhcAAAAgCNTkSeo+/PBDPTkRNStCIt2nTx966y0Wtoi5X0QxbomaSMichQRaJChiPzFEddVVV9HChQvlPhaLhb788kv9X9ELc8MNN+gJjjovTHkJm2GjO+N2Uky0mV57kJVEKe8kytgzlbc1x/G0+gkfGVU3RTcf4RfvsfLiLAerT/YO4Mc67dAAGY9PXSzjDetPZqKCw5fzrIWmXTzV/9z9rIwS2B08BhmzjfNOdyNWHrl283T9h7tx92CcMjNiRKpiA6Do6ttGs/3A2lij0inNdlTGhQmcIccqqqKCON82AEUx/BvmJVYOmSJZvZOn2APEuHjq/pPr+Fhxdt/T/ccraqNsD6uNos18rIPuOBlHOlihla8oh5ym0vYAqqqIr72gWLEHUOQChR6rz28GxcpU/zZljaoQUqf0L9MGwM8+aruKv/Yyp8mvKhsAv8cPROlU0XbYAIAaTg22B0hISChzQrq0tDTSlD8IQvG0fPny0x5XqJG+/vrrSl8fel4AAAAAEFKETc8LAAAAUBvVRuEIkhcAAAAgGHi1k0tl9g9TkLwAAAAAwaAG17zUdFDzAgAAAICQImx6Xq7dfiFZIx303/7TZNsto1l5dPFVt8nYcRWrjeq8x/5FgsdfUDyM/qF6GC2Vcb9zf5fx92vYw+jdK36WceKv7GH0ZS5Pm+zJ5PadW9oYzt0mldPs+L9YFZOVxqqbhD92yjivQYKMNQ97B7Wpw6qlvGg2Hmrv+kvGqxK7Gs5dz8qqnYJEfttEmFmtVMRiHgOeaFbpFGp83RFRhT79i5JcuYb9MxRPogR7rk8Po3hrrk8Vkuph9FdBqk8Po1yPw6d/0ZnwMFJVSOr2Z8TDqKxvbVXlYVQDvxkGVCdQA++jooRzfURNRfyWVarmhcKXsEleAAAAgHCeYbc2gWEjAAAAAIQU6HkBAAAAggCk0oGD5AUAAAAIBlAbBQyGjQAAAAAQUoRNz0vu9PpktTnpyGusHjHXTZFxnTcjZOx4aLeMTf82Ol2qHka7L+Xcb/Ley2T8YuNPZfzHCvYw2nNpNh9o6y4ZvrOzj4xjXextFL/JmFsWNj1pIS5w7Tgm4/SerKKJzcziuAErl8wOVuB0jd0j45+SOsi4hf2QjAuSeXtBgqIqKkjw42EUx18D3IpXkTmmyKeHUXyk4keksRonwcHtpdVDiTZWFWV6+GcWZ+F9dhey11MLR7qMc4tVVZH7tP5FVelh5FHaVT8if/5FZ8TDyI9/kb6qon5BVbV9mceq4DnClHAeSgg1TJqmL5XZP1wJm+QFAAAAqFGI70reSu4fpmDYCAAAAAAhBXpeAAAAgCCAYaPAQfICAAAABAOojQImbJIXx6J1ZDXZ6J8D7pVt1lt41Kzxoytk/P47PI3/pUPuNxxnYd4aGd/1j8UyfuvTQTJOG8VT7if8wgWjbx/vJWNvHheYHvuVC3HjG3PxYeJGY+Hq8XZcoJq8io9b0IRtAFS6puyTcXoCz93f0fWbjH+o01vG9SxcTJuXbJzO3mXiYteCRJ+no+I43r9QKcyNjs6XcbZiA5ASkSPjDA8X5aY4uOj45LpIGSdY1X34eTS1s+VBjjLdf7SJbQ1yPFx0HGHmgt28Ym53li7YVQpzVRuAIrWQV5mKv0ixB1CLbD1KMa1qA+B3qn+xTiny9VeYa7ABKIdtQLkLXStqA1DB4wRUsFtBwtUGAIQQmGE3YFDzAgAAAICQImx6XgAAAICaBGbYDRwkLwAAAEAwwLBRwGDYCAAAAAAhBXpeAAAAgCAg9AGlNAIV3j9cCZvkpWhgN/LanNTqhb2ybcjiX2X86Sd9ZWwzrZSxdSirWAT3rf2njDedO0vG3yzh/Zdcz4qT4r/ZauCj38+Sces6fB3J6/kdmNWWlUMxP+80nDvjimYyTlTUSs0a8bT+lhhWOp0Tu1XG81P78bntR2Wcl8rKnHgzK37yk/0rUYri+XoLNVbtOOMUZY+iNqoTxQqhY14+X4qTVUXHvVEyTrLx9vo+Hl6XYGV7gC359WQc7eRzZxXzfUSa2Zogx+30qSrKK1btAYz3WmBYZ/KpKrIoqqJitV3Z3uPHBkBVG6nqJIHm5w+TX1VRqf35JL7tHPwd/+TKKlQuVRWwATAQzvUOtQYMGwUMho0AAAAAEFIENXn58ccf6dJLL6V69eqRyWSiBQsWGNZrmkaPPfYY1a1bl1wuFw0YMIC2bdsWtOsFAAAAqnySusosYUpQk5fc3Fzq1KkTzZgxw+f65557jqZPn04zZ86kVatWUWRkJA0cOJAKCniIAAAAAAhle4DKLOFKUGteLrroIn3xheh1mTZtGj3yyCN0+eWX621z5syhlJQUvYdm6NChZ/hqAQAAAFATqLE1Lzt37qT09HR9qKiE2NhY6tmzJ61cyQW1pSksLKSsrCzDAgAAANTYgt3KLGFKjVUbicRFIHpaVMTrknW+mDp1Kk2ZMuWUduc9B8ga6SDtClar3Bp7kPe7jVU6t+w82dMj+LDt+4bjXDfxARkf7M2ePdbVf8r40b+GyDjWuV/GcStY7VLYrhFvs4EVTTuvT5VxxGesChJEt2AlktnBx+pb5y8Z/5TaQcZdnd/JeE59VuzUs7CCJrcu5682E78dCuoYfyncinrIlFgo4xyN1TzJMaqqiI9bL0JRFSl+RKl2bj9SzM8/xZZpOPfuwiQZp9mPyDjDzceKMfNQYraiNnKaPL5VRcpgcb6h3ahoKSpm9ZBNyfXdxX48jDx+VEWGdtNp/YvK8jDyq7pR1EMGVVEACiG/EswKq5Aq2H6a66pWv6UaClRFtRjxs62M3FmjsKXG9rwEysSJEykzM1Mue/eyJBkAAACoKaDmpRYmL6mpJ3sgDh3iOUxKXpes84XD4aCYmBjDAgAAAIDaQ41NXpo0aaInKUuWLJFton5FqI569eoV1GsDAAAAKo0ud65MzQuFLUGtecnJyaHt27cbinQ3bNhACQkJ1KhRIxo7diw9+eST1KJFCz2ZefTRR/U5YYYM4ZoSAAAAICTBDLuhmbysXbuWzj//fPl6/Pjx+r8jRoyg9957jyZMmKDPBTN69GjKyMigPn360KJFi8jp5IJMAAAAAIQXQU1e+vbtq8/n4g8x6+7jjz+uL5VlfotvKSbaTO3vGSPbHj18TMazBr0t43HTbpdx0gRWoggSvuMZfh+4g3uAtMITMs5eWkfGMR1cMk5ZwdvsuyBexvX+y0XFnnasujFZWNEiuKAhq4o21W0o4/MiWTr+faNzZdzUys82uwH/qF0m9hfKU8qHvErZuyeJVUSlVUVJ8awqOu7hczSMypDxEU+kjBu5jss4vThOxvXsJ3yqjdo79xnOvaGYlVlxZvZ0ynDzs41WPIyyiji5jVBkM7luu4ydiu9QgVv1LzKOpBYWW32qhIoVlZDBw8iPesifqkjz+FEUleVh5E9V5Hd7U8VVLNWtKgrAp6i2q4pAGCJ+lytj2eWlsKXG1rwAAAAAtZmarDY6fvw4DRs2TBe9xMXF0ahRo/RSD3/s2rVL73DwtcyfP5/v2cf6uXPn1p55XgAAAAAQHIYNG0YHDx6kxYsXk9vtppEjR+olHB999JHP7Rs2bKhvr/LWW2/R888/f8pM+rNnz6ZBgwbJ1yI5qihIXgAAAIBgUEMLdrds2aLXl65Zs4a6d++ut7366qt08cUX0wsvvKALZ0pjsVhOmcbks88+o2uuuYaioniS1JJkpawpT8oDho0AAACAYFBD7QFWrlypJxgliYtAWPWYzWZ9upLysG7dOl09LIabSjNmzBhKSkqiHj160KxZs8qsffUHel4AAACAECarlIefmKxVLIEiLHjq1GHhicBqterTmJRlz6Py7rvvUps2bah3796GdiHA6devH0VERNB3331Hd955p15Lc88991ToGsMmeXkzI42cxVZ6+sY5su2xN4fL+JFxv8q43ic7ZXzvsP6G4xQfZeXMxm/OlnGjjqy0abCU30gHz+UZfuu+vlbG+Q+144NqXDI+qNkWGW+vZ/R1ujj2axmvb9ZFxm3t7OuT2YQVNTFmVuPkNvCtKnLXY5VOjpc9ixKTsw3nPurhfZrE8jPY72GVUNMI9mLa72Y1VQM7b3/IHSvjVk4eH92UV5/PHWksCjteFOlbVeT2rSrK8aMqylPaVVVRoR//otIeRv5UReXzMPKtKvKrENJ3MlWvqqisc1elV1EFCVdVETyMwpAqGjZq2JDVp4JJkybR5MmTT9n8oYceomefffa0Q0aVJT8/X6+NEXOzlUZt69Kliz4diqiLQfICAAAAhJFUeu/evQYrHH+9Lvfddx/ddNNNZR6yadOmej3K4cNsGCwoLi7WFUjlqVX55JNPKC8vj4YP5w4Cf/Ts2ZOeeOIJKiwsrFBvEZIXAAAAIAhUVu5s+t++5fXxS05O1pfTISx4xMSwom6lW7duetvSpUvJ6/XqyUZ5howuu+yycp1L1MXEx8dXeJgLyQsAAAAAJKJWRUiZb731Vpo5c6Yulb7rrrto6NChUmm0f/9+6t+/P82ZM0cvvC1BWP78+OOP9PXXXOZQwsKFC3Vz5bPPPlufKV/IsJ9++mm6//77qaIgeQEAAACCQQ2VSgs+/PBDPWERCYpQGV111VU0ffp0KkEkNFu3btWHh1SEeqhBgwZ04YUXUmlsNhvNmDGDxo0bpyuMmjdvTi+99JKeJFWUsElePpx1AVkcTvrvhJdl29vv83T7w4fwhDmeQzzWt/oTYxdZ/bO4kLXxQj/T/b+yRsb5E9rKWHuN32hXtPlNxpuUYqtrEhbI+LFWxh9oVzu/SU604C62eHOEjLPTfBfmFjXgYtxMLxf41qmTKeNDHo+MW8YfMZxbLcxtGXVIxnvdiTJu4uR99rkTfE73/2d+XRn3idwq46OFPA9AnJmvVXC8kO8v2sTXmFno9FmYm1vk8FmYW+C2+izMLVLa1aJcQbFSsKsWzXoM7UphbrGfwlylYLc8RblnpDC3THuAirb7O0cA1gQVvaYQAkW5wIBXq9ybwlt9byihLPI3IZ0gLS3Np8RZ9KSIxReiN0ednK4yYJ4XAAAAAIQUYdPzAgAAANQoavCwUU0HyQsAAAAQFCo7S65G4QqGjQAAAAAQUqDnBQAAAAgGGDYKmLBJXlJmbyCryU59+vPsgvUKWAWzc1Z7GUf1d8u40cd7DcfZMYqVQY0fWyFj19QWMja9zo/1tvb/lfH3Ldjk6sZ4tim4q9O9Mu7q4HMfbcfT2Zee7j+zJb9p3VqxjL1NWJF01MNxWn2eun+foojpmHSA761YUQhFc7vg7yL2uWjuZLXR7sIkGfeM3C7j9TmNZTwgarOM0wt4IqVEM6uejhaqFgCskhKcKOT7jjRzZ2G2oipymlj9k19k86kqKlTVRsr2bjfHVuJY4Ck2V0xV5DVXzVT/pfZRUZwQyrV9QFP6V1RV5Ifariryd39QFYHyq4VqptqopoNhIwAAAACEFGHT8wIAAADUKIQpr2LMG9D+YQqSFwAAACAYoOYlYJC8AAAAAMEANS8Bg5oXAAAAAIQUYdPzYmrakEwWB6U8ywqVQzd0kHGd936V8dY32I+oxcj9huOcN+i4jPfPZJ+eKa2/kPGLZ10v4xGxS2Q8t9dAGXews4LmcFdWuLhMfH3ZHVh5JMhRPIliWrCv0h4Pt3duyNe7zc2eQD2Sdst4U9FJV1BB12hu31xQX8ZtnMb7XpHTUsaXx66T8bKMVjK+Ina9jA/mx8o4UfEqSs9jj6RoJXU+kc/XGqEogQTZBfxMHIoaKK/A7tPDqLDIt6qoWGlXlUMeRW2kKocEXrfZj1eR77xfK/ajKvLjbWTy53kUiKrI7/aBqI1MVaOuCUTpVAOBeghUCxg2CpiwSV4AAACAGoU+alSZ5IXCFgwbAQAAACCkQM8LAAAAEAwwbBQwSF4AAACAYOAVBWneSu4fnmDYCAAAAAAhRdj0vPx1bxSZXU5qMZJVRd2mseJn/+dxMn793A9k/OK5rBwSPFn3NRlfMuh+GQ9yFcl4XH8+bqKZPXuOnlPsUznk6soKph3FOXx9rXYZzv1bEatrLmzwp4zXFLDfUv/ELTJeld9MxmdHbT+tcmjWkXNlPDB1k+Hc/8rpLeP6ibky3pXNfkiJ9fhbwMEc9jCKNbOa53geq4qiTPz2y8xz+vQpEuQbVEW8rqjQ5tOTyF3oW23kKfKjHFL8i0qjrjOoh/yoikwVVRWVoTbyu66i6iF/yqEyfJVqs3qoLOUQVEXgjIJho4AJm+QFAAAAqFEgeQkYDBsBAAAAIKRAzwsAAAAQDGAPEDBIXgAAAIAgoGlefanM/uFK2CQvi/7xBkVHm+mfwx6QbTMbvC7jDjeP8Vl8e+f1XBQqiDbz6/xLs2S8pzhbxi37/i3jXwq5SPfyzlws/G1+soxHNlsp469z2JpgaMoqw7m/yuos48Gxv8n4X0e5mPbuOmxH8Pi+S/ncjf6Q8Rt7+8p4fNJ/ZfzniRQZp9Y3ZvQ7M7gwN6EJv20OZfF0/7HKszmeo073z+05uVyY61AKdgvzeRuHsr2gSFlnKMAt9D2tv1Zk8V1kq0z1r7ZTkZ92UcCpFOYa2/2MuFa0MLesvz3eihXa+m8PxB6AahwVtSZA8S2o8Yialcr0nmjh+yZHzQsAAAAAQoqw6XkBAAAAahR6zwl6XgIByQsAAAAQDMQMuX7HdcuBFr41Lxg2AgAAAEBIgZ4XAAAAIBhg2ChgwiZ5OeRxUK7HTN3GrpdtX+VFyXjk0O9kPDOzvoyf7veJ4TjPHO0q42kd53H7oQEyntL4cxlP3TeYj9WI2+/acY2M32nOxxm6ebiMF7RjmwL9WFsvkvFDnVmJdOf+NBm/Up870zamp8q4QROHjHceTpRxYitW/xw6ylP6x5h4e0HGcbY5iDLzurwsl0+VUFGWw2d7cY6qHOK3nyfP5lfxQ/m+1UNU4Ke90HeHoklRFRlwlzFFv6JQMuBXhVQxewC/tgGBqIS00FH8YIp+AMSoj5e0SgwbaRg2AgAAAAAIDcKm5wUAAACoUWDYKGCQvAAAAADBQExQV5lxUi18kxcMGwEAAAAgpEDPCwAAABAM9J6TyszzolG4EjbJy8ivbiOz00nbr3lTtjX/+DYZl6ddX/c1q4SmXLNJxrf9zL5Dr1/DSqB1a5vLuFlTVjdt+62hjOu3Yn+gg1vqyDixAyt8BBnb42Uc05VVPvm7eX9XT1b5uPdF+lT8eNNdPhU/dNThu1100Z3wrQYyZ1h9t2ezEkjFkuunPc9/J6C5wPc6c6FvpY65yE+7H1WR2Y9CSF/nqVi7P+GA3/YAVDdQ/ABQO9C8GmmV+GXTkLwAAAAA4IyiS50xw26trXmZMWMGpaWlkdPppJ49e9Lq1auDfUkAAABAreWpp56i3r17U0REBMXFxZW7J+ixxx6junXrksvlogEDBtC2bdsM2xw/fpyGDRtGMTEx+nFHjRpFOTk5tS95mTdvHo0fP54mTZpE69evp06dOtHAgQPp8OHDwb40AAAAoHLDRpVcqouioiK6+uqr6Y477ij3Ps899xxNnz6dZs6cSatWraLIyEj987qgoEBuIxKXTZs20eLFi+nLL7+kH3/8kUaPHl37kpeXXnqJbr31Vho5ciS1bdtWfygiE5w1a1awLw0AAACo3LBPZZdqYsqUKTRu3Djq0KFDuXtdpk2bRo888ghdfvnl1LFjR5ozZw4dOHCAFixYoG+zZcsWWrRoEb3zzjv6KEqfPn3o1Vdfpblz5+rb1ZqaF5H5rVu3jiZOnCjbzGaz3hW1cuVKn/sUFhbqSwmZmZn6v97/ZX5Z2VxpWdJW3vZA9qmqdpwb58a5cW6cu/rPnZXjPWPFsMXkrtQcdcVif3HNWVmGdofDoS9nkp07d1J6err++VxCbGysnqSIz+uhQ4fq/4qhou7du8ttxPbic1301FxxxRXlP6FWg9m/f78+/eCKFSsM7Q888IDWo0cPn/tMmjSpZMpCLFiwYMGCJaBl79691fbZlp+fr6WmplbJdUZFRZ3SJj4Hq4rZs2drsbGxp93u559/1s994MABQ/vVV1+tXXPNNXr81FNPaS1btjxl3+TkZO3111+v0HXV6J6XQBC9NKJGpoSMjAxq3Lgx7dmzR88CwwWRiTds2JD27t2rF0aFC7hv3Hc4gPuuvvsWPS7Z2dlUr149qi6E+ET0VIjRhaq4XpPJON2Dv16Xhx56iJ599tkyjyeGdlq3bk01nRqdvCQlJZHFYqFDhw4Z2sXr1FR2TC5Pd5lIXMLpl7wEcc+47/AB9x1e4L6rhzPxRVckMGI5k9x333100003lblN06ZNAzp2yWey+HwWaqMSxOvOnTvLbUqLbYqLi3UFkr/P9JBMXux2O3Xr1o2WLFlCQ4YM0du8Xq/++q677gr25QEAAAAhQ3Jysr5UB02aNNETEPH5XJKsiJ4yUctSoljq1auXPhoialnFZ7tg6dKl+ue6qI2pVWojMQT09ttv0/vvv693Z4mHkJubq6uPAAAAAFD1iFKLDRs26P96PB49Fos6J4sYXvrss8/0WAxdjR07lp588kn64osv6I8//qDhw4frw28lnQ9t2rShQYMG6QpiMV/bzz//rHdEiGLeig7T1eieF8G1115LR44c0Se+EZXMIqMTUquUlJRy7S+GkMQcMWe68jrY4L5x3+EA7hv3DaoH8ZkrOg1K6NKli/7vDz/8QH379tXjrVu3SkWvYMKECXrngpi3RfSwCCm0+LxWh8c+/PBDPWHp37+/rjK66qqr9LlhKopJVO1W8h4BAAAAAM4YNX7YCAAAAABABckLAAAAAEIKJC8AAAAACCmQvAAAAAAgpKjVycuMGTMoLS1Nr3QWGnIhzapNTJ06lc466yyKjo6mOnXq6HI0Uf2tItw8x4wZQ4mJiRQVFaVXdpee9C/UeeaZZ6RMr7bf9/79++mGG27Q70tYzgvTtLVr11bIkj7UEDLNRx99VJ9HQtxTs2bN6IknnjB4z9SG+xbuupdeeqkuGRXv5xIzu4rco5jsS7j2ignchIfMqFGjDNLWULtvt9tNDz74oP4+Fw7FYhshvy1t4heK9w0qR61NXubNm6fPESNkdevXr6dOnTrp1tylZ/cLZZYvX65/QP/yyy+6vbj4Rb/wwgt1qVoJwhV04cKFNH/+fH178Ut/5ZVXUm1hzZo19Oabb+oOpiq18b5PnDhB55xzDtlsNvrmm29o8+bN9OKLL1J8fHyFLOlDDTGd+RtvvEGvvfaaPteTeC3uU7jR1qb7Fr+34u+U+NLli/Lco/gA37Rpk/734Msvv9QTAyFbDdX7zsvL0/9+i+RV/Pvpp5/qX9Auu+wyw3aheN+gkmi1FGHcOGbMGPna4/Fo9erV06ZOnarVVg4fPqwbYy1fvlx/nZGRodlsNm3+/Plymy1btujbrFy5Ugt1srOztRYtWmiLFy/W/vGPf2j33ntvrb7vBx98UOvTp4/f9V6vVzd7e/7552WbeBYOh0P797//rYUqgwcP1m6++WZD25VXXqkNGzas1t63eK9+9tln8nV57nHz5s36fmvWrJHbfPPNN5rJZNJNbkPxvn2xevVqfbvdu3fXmvsGFadW9rwIsysx/bBqzS0mwxGvhSV3baVksqCEhAT9X/EMRG+M+hzEjIiNGjWqFc9B9DoNHjzYcH+1+b7FrJXCSv7qq6/WhwnFpFFi9unyWtKHKr1799anHP/rr7/017/99hv99NNPdNFFF9Xq+1Ypzz2Kf8WQiXiPlCC2F3/7RE9Nbfo7J4aXxL2G032DEJthNxCOHj2qj5OXnoVXvP7zzz+pNiK8IUTNhxhWaN++vd4m/tgJf6iSX3L1OYh1oczcuXP1bmQxbFSa2nrff//9tz58IoZDH374Yf3e77nnHv1eR4wYIe/N1/s+lO9bOOEKjxSRgAqjVvG7/dRTT+lDBYLaet8q5blH8a9IalWsVqv+Zaa2PAcxRCZqYK677jppzBgO9w3CJHkJR0QvxMaNG/VvpLWdvXv30r333quPb59pV9ZgJ6ji2+XTTz+tvxY9L+JnLmogRPJSW/n444/1KcU/+ugjateune6vIhJ1UbxZm+8bGBG9qddcc41euCySeBDe1Mpho6SkJP0bWml1iXhdUdvtUED4RIgiNeE50aBBA9ku7lUMoQmPidr0HMSwkCi87tq1q/4NSyyiKFcUM4pYfButjfctVCZt27Y1tAmjM2GcVtqSvjbd9wMPPKD3vgjzNqE6ufHGG/WCbKG2q833rVKeexT/lhYkFBcX60qcUH8OJYnL7t279S8tJb0utf2+QZglL6IbXdhti3Fy9VureC0suWsL4huISFyEq6ewFRdSUhXxDIQyRX0OolJffNiF8nMQhl7CsbTE5VQsokdCDCOUxLXxvsWQYGkpvKgDady48SmW9CWUWNKH8n0LxYmoX1ARX07E73Rtvm+V8tyj+Fck7CK5L0H8XRDPSdTGhHriImTh33//vT5NgEptvW9wGrRayty5c/VK/Pfee0+vRh89erQWFxenpaena7WFO+64Q4uNjdWWLVumHTx4UC55eXlym9tvv11r1KiRtnTpUm3t2rVar1699KW2oaqNaut9C5WF1WrVnnrqKW3btm3ahx9+qEVERGgffPCB3OaZZ57R3+eff/659vvvv2uXX3651qRJEy0/P18LVUaMGKHVr19f+/LLL7WdO3dqn376qZaUlKRNmDChVt23UM/9+uuv+iL+NL/00kt6XKKqKc89Dho0SOvSpYu2atUq7aefftLVeNddd50WqvddVFSkXXbZZVqDBg20DRs2GP7OFRYWhvR9g8pRa5MXwauvvqp/gNntdl06/csvv2i1CfGL7muZPXu23Eb8Ybvzzju1+Ph4/YPuiiuu0H/xa3vyUlvve+HChVr79u31xLx169baW2+9ZVgvJLWPPvqolpKSom/Tv39/bevWrVook5WVpf9sxe+y0+nUmjZtqv3f//2f4cOrNtz3Dz/84PP3WSRv5b3HY8eO6R/aUVFRWkxMjDZy5Eg9OQjV+xbJqr+/c2K/UL5vUDlM4n+n650BAAAAAKgp1MqaFwAAAADUXpC8AAAAACCkQPICAAAAgJACyQsAAAAAQgokLwAAAAAIKZC8AAAAACCkQPICAAAAgJACyQsAoFzs2rWLTCaTbsEAAADBBMkLACHCTTfdpCcPYhHeTcKA8oILLqBZs2ZJn5+qPNeQIUOq9JgAAFBVIHkBIIQYNGgQHTx4UO8F+eabb+j888+ne++9ly655BLdSRcAAMIBJC8AhBAOh0N3F65fvz517dqVHn74Yfr888/1ROa9997TtxEOu7fccgslJydTTEwM9evXj3777Td5jMmTJ1Pnzp3pzTffpIYNG1JERITu2puZmSnXv//++/pxS3p6li1bJvf/+++/9aRJ7NepUydauXJlEJ4EACCcQfICQIgjkhORRHz66af666uvvpoOHz6sJzTr1q3Tk5z+/fvT8ePH5T7bt2+njz/+mBYuXEiLFi2iX3/9le6880593f33368nMyW9PGLp3bu33Pf//u//9G1E7UvLli3puuuuQ68PAOCMguQFgFpA69at9aGkn376iVavXk3z58+n7t27U4sWLeiFF16guLg4+uSTT+T2BQUFNGfOHL0H5rzzzqNXX32V5s6dS+np6RQVFUUul0v28ojFbrfLfUXiMnjwYD1xmTJlCu3evVtPhgAA4EyB5AWAWoAwhxfDO2J4KCcnhxITE/UkpGTZuXMn7dixQ27fqFEjfeiphF69eulFv1u3bj3tuTp27CjjunXr6v+Knh4AADhTWM/YmQAA1caWLVuoSZMmeuIiEgq1RqUE0ftSFQilUwkiYRJUtdoJAADKAskLACHO0qVL6Y8//qBx48ZRgwYN9KEfq9VKaWlpfvfZs2cPHThwgOrVq6e//uWXX8hsNlOrVq3012KYyOPxnLF7AACAioDkBYAQorCwUE9ORGJx6NAhvdh26tSpulR6+PDhegIihoDEHC3PPfecXpcikpSvvvqKrrjiCr0ORuB0OmnEiBF6PUxWVhbdc889epGuqG8RiMTn22+/1YeRxBBUbGxskO8cAAAYJC8AhBAiWRHDQqJnJT4+XlcZTZ8+XU9EROIi+Prrr3VF0MiRI+nIkSN6QiKKcsWkdiU0b96crrzySrr44ot1FZJIfl5//XW5/tZbb9WHnkSyI4aifvjhhzJ7cgAA4Exi0kSlHwAgbBDzuCxYsADT/AMAQhaojQAAAAAQUiB5AQAAAEBIgWEjAAAAAIQU6HkBAAAAQEiB5AUAAAAAIQWSFwAAAACEFEheAAAAABBSIHkBAAAAQEiB5AUAAAAAIQWSFwAAAACEFEheAAAAABBSIHkBAAAAAIUS/w8numZWEAS0GwAAAABJRU5ErkJggg=="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 358
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.221867Z",
     "start_time": "2025-03-24T10:58:40.214360Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#随机input，调用TransformerEmbedding\n",
    "config = {\n",
    "    \"vocab_size\": 100,\n",
    "    \"d_model\": 128,\n",
    "    \"pad_idx\": 0,\n",
    "    \"max_length\": 64,\n",
    "    \"dropout\": 0.1,\n",
    "}\n",
    "# 生成一个形状为 (2, 50) 的随机整数张量，其中每个整数的取值范围在 [0, 100) 之间\n",
    "input_ids = torch.randint(0, 100, (2, 50))\n",
    "embeds = TransformerEmbedding(config)(input_ids)\n",
    "embeds.shape"
   ],
   "id": "185b35d0c9513f10",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 50, 128])"
      ]
     },
     "execution_count": 359,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 359
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Transformer Block",
   "id": "439190c4e18c8eba"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## scaled-dot-product-attention",
   "id": "9ade35b913a7f3df"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.235502Z",
     "start_time": "2025-03-24T10:58:40.223373Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from dataclasses import dataclass\n",
    "from typing import Optional, Tuple\n",
    "\n",
    "Tensor = torch.Tensor\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class AttentionOutput:\n",
    "    hidden_states: Tensor\n",
    "    attn_scores: Tensor\n",
    "\n",
    "\n",
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "\n",
    "        # 隐藏层大小\n",
    "        self.hidden_size = config[\"d_model\"]\n",
    "        # 多头注意力的头数\n",
    "        self.num_heads = config[\"num_heads\"]\n",
    "        assert (\n",
    "                self.hidden_size % self.num_heads == 0\n",
    "        ), \"Hidden size must be divisible by num_heads but got {} and {}\".format(\n",
    "            self.hidden_size, self.num_heads\n",
    "        )\n",
    "        # 每个头的维度\n",
    "        self.head_dim = self.hidden_size // self.num_heads\n",
    "\n",
    "        #第二个self.hidden_size可以*系数\n",
    "        self.Wq = nn.Linear(self.hidden_size, self.hidden_size, bias=False)\n",
    "        self.Wk = nn.Linear(self.hidden_size, self.hidden_size, bias=False)\n",
    "        self.Wv = nn.Linear(self.hidden_size, self.hidden_size, bias=False)\n",
    "        # 输出层\n",
    "        self.Wo = nn.Linear(self.hidden_size, self.hidden_size, bias=False)\n",
    "\n",
    "    def _split_heads(self, x: Tensor) -> Tensor:\n",
    "        # 输入形状：[batch_size, seq_len, hidden_size]\n",
    "        bs, seq_len, _ = x.shape\n",
    "        x = x.view(bs, seq_len, self.num_heads, self.head_dim)\n",
    "        # 输出形状：[batch_size, num_heads, seq_len, head_dim]\n",
    "        return x.permute(0, 2, 1, 3)\n",
    "\n",
    "    def _merge_heads(self, x: Tensor) -> Tensor:\n",
    "        # 输入形状：[batch_size, num_heads, seq_len, head_dim]\n",
    "        bs, _, seq_len, _ = x.shape\n",
    "        # 输出形状：[batch_size, seq_len, hidden_size]\n",
    "        return x.permute(0, 2, 1, 3).reshape(bs, seq_len, self.hidden_size)\n",
    "\n",
    "    def forward(self, querys, keys, values, attn_mask=None) -> AttentionOutput:\n",
    "        querys = self._split_heads(\n",
    "            self.Wq(querys))  #(batch_size, seq_len,hidden_dim)-->[batch_size, num_heads, seq_len, head_dim]\n",
    "        keys = self._split_heads(self.Wk(keys))  #[batch_size, num_heads, seq_len, head_dim]\n",
    "        values = self._split_heads(self.Wv(values))  #[batch_size, num_heads, seq_len, head_dim]\n",
    "\n",
    "        # calculate attention scores\n",
    "        qk_logits = torch.matmul(querys,\n",
    "                                 keys.mT)  # 计算注意力分数，matmul是矩阵乘法，mT是矩阵转置,qk_logits是[batch_size, num_heads, seq_len, seq_len]\n",
    "        # print(querys.shape[-2], keys.shape[-2])  #3 4\n",
    "        if attn_mask is not None:\n",
    "            attn_mask = attn_mask[:, :, : querys.shape[-2], : keys.shape[-2]]\n",
    "            qk_logits += attn_mask * -1e9  # 给需要mask的地方设置一个负无穷\n",
    "        attn_scores = F.softmax(qk_logits / (self.head_dim ** 0.5), dim=-1)  # 计算注意力分数\n",
    "\n",
    "        # apply attention scores\n",
    "        embeds = torch.matmul(attn_scores, values)  # softmax后的结果与value相乘，得到新的表示\n",
    "        embeds = self.Wo(self._merge_heads(embeds))  # 输出层 [batch_size, seq_len, hidden_size]\n",
    "\n",
    "        return AttentionOutput(hidden_states=embeds, attn_scores=attn_scores)\n",
    "\n",
    "\n",
    "mha = MultiHeadAttention({\"num_heads\": 2, \"d_model\": 2})\n",
    "query = torch.randn(2, 3, 2)  # [batch_size, seq_len, hidden_size]\n",
    "query /= query.norm(dim=-1, keepdim=True)  # 归一化\n",
    "key_value = torch.randn(2, 4, 2)\n",
    "print(f'key_value.shape {key_value.shape}')\n",
    "outputs = mha(query, key_value, key_value)  #最终输出shape和query的shape一样\n",
    "print(outputs.hidden_states.shape)\n",
    "print(outputs.attn_scores.shape)"
   ],
   "id": "3be37f2ce3677e42",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key_value.shape torch.Size([2, 4, 2])\n",
      "torch.Size([2, 3, 2])\n",
      "torch.Size([2, 2, 3, 4])\n"
     ]
    }
   ],
   "execution_count": 360
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.324014Z",
     "start_time": "2025-03-24T10:58:40.316507Z"
    }
   },
   "cell_type": "code",
   "source": [
    "mha = MultiHeadAttention({\"num_heads\": 2, \"d_model\": 2})\n",
    "query = torch.randn(2, 3, 2)  # [batch_size, seq_len, hidden_size]\n",
    "query /= query.norm(dim=-1, keepdim=True)  # 归一化\n",
    "key_value = torch.randn(2, 4, 2)\n",
    "print(f'key_value.shape {key_value.shape}')\n",
    "outputs = mha(query, key_value, key_value)  #最终输出shape和query的shape一样\n",
    "print(outputs.hidden_states.shape)\n",
    "print(outputs.attn_scores.shape)"
   ],
   "id": "ccc2e2a0f5fe2ecd",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key_value.shape torch.Size([2, 4, 2])\n",
      "torch.Size([2, 3, 2])\n",
      "torch.Size([2, 2, 3, 4])\n"
     ]
    }
   ],
   "execution_count": 361
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.358414Z",
     "start_time": "2025-03-24T10:58:40.352520Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#随机一个2阶张量，在-1维度计算softmax\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "x = torch.randn(2, 3)\n",
    "x_softmax = F.softmax(x, dim=-1)\n",
    "print(x_softmax)"
   ],
   "id": "abcc387ea56973b7",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.5712, 0.1760, 0.2528],\n",
      "        [0.8301, 0.1050, 0.0649]])\n"
     ]
    }
   ],
   "execution_count": 362
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.645080Z",
     "start_time": "2025-03-24T10:58:40.521666Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# plt.subplots() 用于创建子图网格，其维度基于 outputs.attn_scores.shape[:2]。子图的行数和列数似乎由 outputs.attn_scores 的前两个维度确定。\n",
    "fig, axis = plt.subplots(*outputs.attn_scores.shape[:2])\n",
    "for i in range(query.shape[0]):\n",
    "    for j in range(outputs.attn_scores.shape[1]):\n",
    "        # axis[i, j].matshow(outputs.attn_scores[i, j].detach().numpy())：此行使用 Matplotlib 的 matshow 绘制每个 i 和 j 的注意力分数热图。detach().numpy() 将 PyTorch 张量转换为 NumPy 数组以进行可视化。\n",
    "        axis[i, j].matshow(outputs.attn_scores[i, j].detach().numpy())\n",
    "        for x in range(outputs.attn_scores.shape[2]):\n",
    "            for y in range(outputs.attn_scores.shape[3]):\n",
    "                # axis[i, j].text(y, x, f\"{outputs.attn_scores[i, j, x, y]:.2f}\", ha=\"center\", va=\"center\", color=\"w\")：此代码在热图上叠加文本，显示 (x, y) 位置处的注意力分数。格式化部分 f\"{outputs.attn_scores[i, j, x, y]:.2f}\" 确保以两位小数显示注意力分数。文本以白色居中显示在 (y, x) 坐标处。\n",
    "                axis[i, j].text(y, x, f\"{outputs.attn_scores[i, j, x, y]:.2f}\", ha=\"center\", va=\"center\", color=\"w\")\n",
    "fig.suptitle(\"multi head attention without mask\")\n",
    "plt.show()"
   ],
   "id": "97810e361373e0bc",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhUAAAG6CAYAAAC/RrTYAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAlx9JREFUeJzs3QV4U1cbB/B/2tRdaJG2uDsV3N1dh7ttg43BYMKYIBMGbNgEGDCGy8Zwd7cWKtRoS92SeiP3e84JTZs2QGEhofne3/Nc6L05ucnJTU7ee857bkSCIAgghBBCCPmPTP7rDgghhBBCGAoqCCGEEKITFFQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJCirIGzV+/HhUqVKlVGW/+OILiESil5br0KEDGjRogLcJqyOr6/+LVzmub4PSvreKlk1OTn7jz8uY0OtGGAoqiF5lZ2fzxufcuXOGfipvtaVLl+LgwYMltl+5coW/funp6W/8OcTGxvLHunfvHv6fXmNDevToEX/NIyMjDf1UCHktFFQQvQcVS5Ys0RpUfPrpp8jJyTHI8ypLQQV7/fQVVLDH0hZU/PrrrwgODkZZoe299bYGFew1p6CClFViQz8BQgqIxWK+kLefmZkZyhJ6bxGiH9RT8X889hkSEoLRo0fDwcEB5cqVw2effQb2o7XR0dHo378/7O3tUb58efzwww8a99+yZQu/f/GzKdb7wLY/b2iDlWePw7CzMVaWLez5FH1er3JW17FjR1hbW6NSpUr49ttvS5TJy8vD4sWLUaNGDVhYWMDT0xPz58/n24vavHkzOnXqBDc3N16uXr16WL9+fYn9sdfn66+/hoeHB39c9vgPHz4s9XP+/vvv0apVK7i4uMDKygre3t7Yu3evRhn2GmRlZeGPP/5Qv0Ysh4G9Ph999BEvU7VqVfVtRY/D9u3b+T7Zvp2dnTFixAh+PLXlpLzo9WPH0NfXl/89YcIE9WOxY/+8nAr2nD/88EP+GrPXsHbt2ry+xX8Ime1n9uzZvJeAPQ9Wtn79+jh27NgLXzu2H1dXV3zwwQfqbUqlEo6OjjA1NdXovVmxYgUPIjIzM7W+t573GhfF9se2sf2zzwh7HVhPW1FyuRxfffUVqlevzuvBXpNFixaVeH8VfZ8/LxeHvbZDhw7lf7PjUvC8XjRUyO5ra2uLqKgo9OnTh//NjuXatWv57f7+/vx9bWNjg8qVK2PHjh0a909NTcW8efPQsGFDfl/2me/Zsyfu379f4rF++uknfpzY+8XJyQk+Pj4l9lfckydP+GePHeeEhIQXliXGgYKK/2PDhw/njfLy5cvRvHlz/mW5atUqdO3alTdMrGFmDQJrdC5cuPCfH48FFAVf1AMHDsS2bdv4MmjQoFfeV1paGnr06IHGjRvzoKdOnTpYsGABjh49qi7D6tavXz/+xda3b1/eKA4YMAA//vgjr3tR7HmxRpd9IbD9sS/GmTNnqhvnAp9//jkPvtjjfvfdd6hWrRq6devGv6BKY/Xq1WjatCm+/PJL3v3OvvjYF8m///6rLsNeE/YF1bZtW/VrNG3aNP46jRw5kpdhdSi4rSBQ++abbzB27FjUrFkTK1euxJw5c3D69Gm0a9euxHDJy16/unXr8ufITJ06Vf1YbF/P+8JnrzV7Xmy/7PFZUMGCoKJBQIFLly7x15cFPSyYyc3NxeDBg5GSkvLc1459wbZu3VrjvfjgwQNIJBL+9+XLl9XbL168yF9n9kWpzfNe46KGDRuGjIwMLFu2jP/NvvRZMFzU5MmT+XuiWbNmvO7t27fn5Vm9XhV7bd977z3+N3sfFjwvdixeRKFQ8ECAvWfZa8kCFRa0sefLjgX78mefZTs7O/7+iIiIUN83PDycB3csIGHHjB0vFoiwerDhr6LDXey5sWCbtRHsdWjSpAmuX7/+3OcVFhbG68QelwVG7u7ur/yakDJIIP93Fi9ezE4dhalTp6q3yeVywcPDQxCJRMLy5cvV29PS0gQrKyth3Lhx6m2bN2/m94+IiNDY79mzZ/l29n8Bdr/KlSur15OSkngZ9hye97xepn379rzc1q1b1dvy8vKE8uXLC4MHD1Zv27Ztm2BiYiJcvHhR4/4bNmzg9798+bJ6W3Z2donH6d69u1CtWjX1emJiomBubi707t1bUCqV6u2LFi3i+yv6Gj1P8cfJz88XGjRoIHTq1Elju42Njdb9fffdd1pf+8jISMHU1FT45ptvNLb7+/sLYrFYY3tpX7+bN2/ycux4F1f8uB48eJCX/frrrzXKDRkyhL+nQkND1dtYOfY6Ft12//59vv2nn34q8VjF68/qKZVK+fqaNWv48/Dz8xMWLFjAtykUCsHR0VGYO3fuC99bz3uNC8pOnDhRY/vAgQMFFxcX9fq9e/d4ucmTJ2uUmzdvHt9+5swZjTpre8+z5170OezZs6fEZ+hF2H1Z+aVLl5b4zLLXfefOnertQUFBJZ5Hbm4uf72KYu8tCwsL4csvv1Rv69+/v1C/fv0XPpeC1419xgMDA4WKFSsKvr6+QmpqaqnqQowD9VT8H2NnWQVY9zE7o2Ht36RJk9TbWdcvO+NkZzRvE3YGyoZuCpibm8PPz0/jee7Zs4ef5bGzcDbNrWBh3cHM2bNn1WXZcEEBdubLyrGzNba/gjPhU6dOIT8/H++++65GVzrrESitoo/DegvYvtnZ8p07d/Bf7N+/n/fMsDPqonVlw1es56JoXUv7+r2KI0eO8PdQwZl2ATYcwt5TRXuQmC5duvAhgwKNGjXiXe8ve3z2WrEzc5awWtAjwbaxhf3NBAQE8J4Ztu2/mD59eonHZj0pUqlUXWemeE8MqzNTtPdJn5/lgs8sG/Jg74cCbBu7rehrzHprTExUXwPsdWX1Y+8NVrboe5LdLyYmBjdv3nzpc2GvP/vssB4T9plhQyXk/wcFFf/HvLy8NNbZuLGlpSUfty6+nX0Bvk1YTkPx/AvWeBV9no8fP+b5Dmx4oOhSq1YtfntiYqK6LOs6Z190rCFmDSgrx7qgmYKggo0PM+xLuihWtrQN5+HDh9GiRQv+OrOch4IhoYLHeF2sruzLmz234vUNDAzUqGtpX79XwV6bihUr8q7uogq67gteu+e990r7+GyYgY3pFwQQBUEF62a/desWH0YpuK1NmzavVZfnPceCY1zwHFmd2BcyGyIsigVy7D1UvM5vCnsvFQyBFf3MajvGxT/LLBBlwzbsfcMCDPbZZ/sqOqzEsKExFmywwJOVnTVrlsZwU1FsqJG9D44fP84DRfL/hdKh/4+xM8vSbGOKJts9L5mSnenoS2meJ2swWQIaGyvWho1BF4z9du7cmfdosLJsOztzZ2eirMFl+9EF9mXH8g7YF+C6detQoUIFPouCJYm+LOHtZdhzZMeF9Qhoe22K5xaU5vV7k1738dnrxfJ/WF5FaGgo4uPjeVDBxutlMhkf42evMzuWxb9o39RzfJXk4jfxmXne8yzN82d5PSxHaOLEiTzhlAW6LFBivW9F3/csOGRTiFlQzBJq9+3bx9/DLJ+keJ4Jy41hCbB//vlniTwVYvwoqCCvrOCMrXjyX2nOzP5LA/yqWPc6y2JnAcOLHveff/7h2fp///23xtlp8SEDlshZ0CvAEjQLJCUlleoMnzXE7KySncGxs8ICLKgo7nnP93nbWV3ZlwWbFVLQE/NfvcqxYq8N6+pmiY1FeyuCgoLUt+sKCyJY4iF7PHZmzQII9lzZzAQWULCFJR6+6fciqxP74mXvh6LJlGyWA/tsFK0z+8wU/7ywobS4uDidPqdXxWYesZkmv//+u8Z29lyL91iyXjyW4MwW9txZ4jBLDl64cCF/XxdgCcwsAZkl4rL3wqhRo/RWH2J4NPxBXlnBWHjRLHx2xvXLL7+89L6s65rRx8Wb2Hjy06dPeeZ6cexCSAUzNgrO6IqewbGu3+Jf9mx4hJ0ps1kkRcuybPjSYI/DvjSKnp2y6aDaLsDEGnBtrxHbzhS/jTXwbP/srLH4mTRbf9Gsiud53mNp06tXL16vn3/+WWM76+lhdWazE3QZVLAgkL3ubIij4Iu4YCYHm7VQmnyK573GpcXqrO34F/SM9e7dW+MzU3wGFfu8FO+peJXXXBfYe6b4+4XlIrHPTVHF3z+sJ4/NBGH3ZT1ERbHjweo2ZMgQjBs3jgfr5P8H9VSQV8bOCFleADtDYfPcWZfpzp07+Zz90iQqssZo165d/Iya3ZfNYX8Tv+UxZswY7N69myfcsV4HNh2RNeLs7JltZz0GLDmVTQlljSQbC2bdtezaBiwQYdesKHomybrT2fRaNmWQnQmzL5W7d+/yIYfiZ3XasC8Z9oXDpvmxszeW58CmrLIxeTaGXRS71gQ7E2flWa4C64Fg3f5sO/PJJ5/waYssyGHPm31psSnB7JiwQIVNnWVniWz64IEDB/i0UPbcXwXbJ8sN2LBhA98X+8Jjz4E9l+LYc2BnvOx5scdnU1VPnDiBQ4cO8a70okmZ/1XLli35mTDrjmf1KsCGlQqmLJcmqHjea1xarI7sS5N9gbIggCUn3rhxg3f9s9efvR5FEynZ+5ANDbAp26wHjb3/ir9v2DRN9kXPemJYYMt6tAqun/ImsPcxmzrMrsHBrp/CppOyYYuiPXEM+4ywXBH2GWJDTSxPhwWQ7D1dPI+GYUMo7Jop7HVgwT0bSixIkCZGztDTT4j+FZ36VXx6GptmVxybglh8OllYWJjQpUsXPvXM3d2dT6s8efLkS6eUMleuXBG8vb35tMKiU9xeZUqptult2h6LTdlcsWIFL8+eq5OTE3/sJUuWCBKJRF3u77//Fho1aiRYWloKVapU4ffZtGlTiembbPodu2+FChX4tL0OHToIAQEBJaYGPs/vv/8u1KxZkz+XOnXq8Oma2urNpv+1a9eOP0bx6apfffWVUKlSJT5dtvjz27dvn9CmTRt+HNnCHmPWrFlCcHDwa71+hw4dEurVq8enpRadXqqtbEZGBp/GyaYSmpmZ8XqyKaBFp98ybD/sORVX2teQYVMV2X6uX7+u3hYTE8O3eXp6lij/Kq/x8z4f2qZSy2Qy/n6oWrUqrzN77IULF/KpmkWx9w2b8urq6ipYW1vz6cpsSq22Ov/66698KjObOvuy6aWv8pll2OOxKdEF2PP88MMP1e/n1q1bC1evXuX3Z0uBjRs38teKTall793q1asLH330kcZnSNvrxqZQs/3Y2toK165de249iPEQsX8MHdgQQgghpOyjnApCCCGE6AQFFYQQQgjRCQoqCCGEEKITFFQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJow4q2I81ValShf8sL/uhIPZjP8aC/eIh+xEn9kNI7FcBtf3SZVnFfrDL19eX/1AR+yEl9qNE7MejjAX70atGjRrB3t6eL+wHstiPkpG3h7G2HcbcbjDUdhie0QYV7FcwP/jgAyxevBh37tzhvyjYvXt3/suQxoD9bDerE2v8jM358+cxa9YsXLt2DSdPnuQ/rcx+JbHgp8rLOg8PDyxfvhy3b9/GrVu3+K839u/fHw8fPjT0UyNG3nYYc7vBUNvxFhCMlJ+fn8YvIbJfCWS/nrhs2TLB2LDDeODAAcFYJSYm8jqeP39eMFbs11N/++03Qz8N8n/Udhh7u8FQ26F/RtlTkZ+fzyO5Ll26qLeZmJjw9atXrxr0uZFXJ5FI+P/Ozs4wNgqFAjt37uRnUqwrkxgWtR3GhdoO/RPDCCUnJ/MX3N3dXWM7Ww8KCjLY8yKvTqlUYs6cOWjdujUaNGgAY+Hv788bgtzcXNja2uLAgQOoV6+eoZ/W/z1qO4wHtR2GYZRBBTEebHw0ICAAly5dgjGpXbs27t27x8+k9u7di3HjxvHx4LepcSCkLKO2wzCMMqhwdXWFqakpEhISNLaz9fLlyxvseZFXM3v2bBw+fJhnrLMEJWNibm6OGjVq8L+9vb1x8+ZNrF69Ghs3bjT0U/u/Rm2HcaC2w3CMMqeCvejsxT59+rRGVxhbf5vGnoh2LIeMNQqsW+/MmTOoWrUqjB17f+bl5Rn6afzfo7ajbKO2w/CMsqeCYVPCWLeQj48P/Pz8sGrVKp7QMmHCBBiDzMxMhIaGqtcjIiJ4lxhLSPLy8kJZ77bcsWMHDh06xOebx8fH8+0ODg6wsrJCWbdw4UL07NmTH6eMjAxe13PnzuH48eOGfmrEyNsOY243GGo73gKCEfvpp58ELy8vwdzcnE8Tu3btmmAszp49y6dKFV/GjRsnlHXa6sWWzZs3C8Zg4sSJQuXKlfn7sly5ckLnzp2FEydOGPppkf+DtsOY2w2G2g7DE7F/DB3YEEIIIaTsM8qcCkIIIYToHwUVhBBCCNEJCioIIYQQohMUVBBCCCFEJyioIIQQQohOUFBBCCGEEJ2goIIQQgghOmH0QQW7fOkXX3zxVl3GVJeofmWbsdevrDL240L1K9vy3uL6Gf3Fr6RSKb9EK/tFN3t7exgbql/ZZuz1K6uM/bhQ/co26VtcP6PvqSCEEEKIflBQQQghhJCy+Sul7GdaY2Nj+S/IiUQivXQTFf3f2FD9yjZ914+NdrJfN6xYsSJMTMrOOQW1G7pF9SvbpG9xu6H3nIqYmBh4enrq8yEJIcVER0fDw8MDZQW1G4SUjXZD7z0V7EyDqTn1c5iaW8IYCa0kMGauv1vBmJnmKGCs5PI8XLnxrfpzWFYUPN82uyZBbG0OY9TaNQzG7NgP7WDMHP1TYazkijycD11bqnZD70FFQdclCyhMLYw0qLDOhTETi43zuBUwFRtvUFFAH0MIb+L5soBCbGMBY2RpawZjJjYz7nZDbGqc78tXbTfKzqAqIYQQQt5qFFQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJCioIIYQQohMUVBBCCCFEJyioIIQQQohOUFBBCCGEEJ2goIIQQgghOkFBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE6IUcaMaNUYEzp4w9XOBsFxSVh64CwCohO0lh3cvAH6eddDjfIufP1RTCJWH71Uonw1N2fM7d0GPtU8YGpqgvCEFMz54zDi0zOgbyOr+mFCjVZwtbBFsDQBSx8cgX/6U61lu1Soiym12sLLxhlikSmislKwJfQK/ol5oC4zs3YH9KzUAOWtHCBTKvBIEovVgafhn6Z9n2/agL5NMWJIczg72SA0PBFr1p1CUEic1rK9ezRG9y71UbVyOb4eEhqPXzdf0CjftnUt9OvVBLVqloeDvRUmz9zM92so/Qd4Y9iI5nB2tkVYWAJ+Wn0CwUHa69erTxN0694QVaq68vWQ4Hj8/uu5EuXHT2zHy9raWiDAPwarVx7D06dpeqmPMelXqTWGenaCs7kdwrJisTZkP4IzorSW7VmhBbqW90UVm/J8/XFGDDaF/6tR/qM6I9Gtgp/G/W6mBGLRg19gCN7OvdHSdRBsxU5IyI3A8biNiM0J0Vq2qVN3NHTshHKWlfl6fE4oziZsLVHexcIDnd0nwMumAUxEpkjOjcLe6GWQypKgb4O7NsHoPj5wdrBBaFQSfvjjDB6FxWst279jQ/RsWw/VPFWfreCIBKzfdUmj/GfTuqN3+wYa97t6PwJzV+yHIfQd2QJDJrSFk6stwoPjsW7pPwjxj9FatscQH3Tp1wyVa7jz9dBHT7F59Ynnln/38/7oPbw5Niw/jIPbrrx9PRVr165FlSpVYGlpiebNm+PGjRvQhx6Na2F+v3ZYf/Iahq76E8Gxydg4ZRCcba20lvet7oEj94IwccNejP5pJ+IlGfhl6iC42duoy3i6OGDrrGGISEzDhPV7MPiHbdhw8jry5XLoW4+K9TG/fnesCz6Hoec3IlgSj40tx8DZvPD5FiXJz8EvIRfwzoXfMOjsOhyIuoevmw5A63LV1WWeZKbgG/8jGHh2HcZc+h1Ps9Pxa8uxcDK3hr51bFcHM6d0wpbtlzFl9haEhSfiu2+GwdFB+3Np0sgTp88FYu6CvzBr7jYkJmXg+6XD4Opiqy5jaWkG/4cx+GXTORhah451MX1WZ2z94xKmT9mEsLBErPh+BBwdtdevcRMvnDn9EB/O+RPvztyKpCQpvv1+JFxdC+s3YmQLDBzkg1U/HMXs6VuQmyvD8u9HwMzcFGWRodqO9m5NMK3GAGyPPI4Zt35AeGYsljWeBkezwte6qMaONXA24Q4+urcW799ZjaS8NCxvPB0u5g4a5W6kBGLY5c/Vy9JH22AI9ezbomv5ybiY+Bd+C3ufBxUjq3wJa1PN51ugsk1DPJScx/aIhdgSNo8HCaOqfAk7seoEjHEyL49xVb9Fcl4MtkUsxK+hs3ExaSfkynzoW5cWtfH+6Pb4bf9VjPtkGx5HJWHVx4PhZK+97W9WzxMnrwRh1te7MWXxX0hIycDqjwejnJPm8b56LwK9ZqxXL5///C8MoV2Phpgyvxe2rzuN2UPXIjw4Dt9snAAHZ+1tfyPfajh35D4WTPwNc9/ZgKR4CZb+MgEubvYlyrbqXA91GnsiOUGih5q8RlCxa9cufPDBB1i8eDHu3LmDxo0bo3v37khMfPNnh2PbN8Pe6wE4ePMRwhNS8eW+U8iVyTHQVzPaLPDxjmPYdeUBgmOTEJGUhsW7T8JEJEKLml7qMu/1aI2LQZFY+e9FBMUmITpFgnOPwpGamQN9G1ejFfY+uY2DUfcQlpGEJfcPI1chw6DKTbWWv5kSidNxQQjPTEZ0dhq2h19DiDQBzVxUZx/Mv0/9cS0pHDHZaXyf3wYch52ZJWrZqyJcfRo6yBf/HruPYyf98SQqBSt/Oo7cPBl6dW+otfw33x7GocN3ec9DVEwqvlt1FCKRCM2aFNbv5OmH2LrjCm7fjYShDRnmhyOH7+H40Qd48iSZBwJ5uXL06NVYa/llX/+Nvw/eQVhoIqKjUvDDt0cgMhGhqXcVdZlBQ/2wfdtlXLn8GOHhSVix9B+4utihTZvaKGsM2XYM9uyAo7FXcTz+BqKyE7A6eA/ylPnoXqG51vLLA7fjn9jLCMuMRXR2IlYG7eLvvaZONTXKyZRypOVnqJdMuf7bDaa56wDcTTuO++mnkJwXjSOxayFT5qGJU1et5Q/GfI/bqUd48JGSH4PDT3+CCCaoYlv4Xu3gNhZhmbdwJmEzEnLDkZYfj8cZN5Ct0M+XU1Eje3nj0Fl//Hv+ISKfpmLF7yd529Gnvfa2Y/HaI9h36j4eP0nCk9hULP3lBG/7fRoUtv1MvlyBVEm2esnIyoMhDBrXBsf23sTJg3cQFZaIn5YcQl5uProP8tZa/tsFu3F453WEB8UhJiIJqz7fz9uOJi0KTygZFmTMWNQX387fDYVc+XYGFStXrsSUKVMwYcIE1KtXDxs2bIC1tTU2bdqEN0lsaoJ6ldxxLaSw+1EQgGuPo9C4coVS7cPSXAyxqSkk2bl8XSQC2tWtisikNGycMhDnv5iGHe+NQKf6mgdGH8xEpqjnUAFXk8LV2wQIPCBo7ORZqn00d62KKrauuJUS+dzHGFrZG1JZDh9a0Sex2AS1a5bH7btPNI4fCwbq1a1Uqn1YWJjx/WRkqI7f24Q9r1q1KuDO7UiN+t25HYF69V+xflJV/SpUcISLiy3fR4GsrDwEBsaWep9vE4O1HSJT1LL1wJ20EI3P1p3Ux6hnXxigvoiFqTnEIhNkyLNL9Gjsbv0lNjVfiPdqDYGdWP89gCYiMSpY1UBE5r0iWwVEZt5DJes6pdqHmYkFH97IURQM+YpQw84HKXmxGFn5S8ytsx0Tqv2AWnYtoG+s7a9d1R03AzTbfrbesGYp234LMUzFJpBmarYdzep64Mj6Gdj1/QTMn9gZ9raW0DexmSlq1quIu1dD1dsEQcDda2Go21gzCHoeC0vWdpgiQ1L4/mRB8EfLh2Lv5ot4Eqa/IeFXCiry8/Nx+/ZtdOnSpXAHJiZ8/erVq3iTnGys+JsrJVPzQ52SkQ1X+9J9kD/o3RZJkkxcfax6czrbWsPG0hyTOvniUlAkpv6yH6f9w7BqXF/4VNNvo+1oYQ2xiSlS8jI1trN1V0vtXbSMrdgCN3svwr2+n2N9i3ew1P+IRmDCtHevxcvc6fspxlZviSlXtiI9X/N1fNMc7K15vkpqepbG9rT0bJ5fURrTJrZHckrmW9ErUZyDgzVvtNLSitUvLQvOz+nCLG7K9I5ISc7E7WdBhNOz+6WlltxnwW1lhSHbDgczG5iamPKehKLSZBlwsijZXazN5Op9kJIv1QhMbqYG4dvAPzH/3nr8FvYPGjlWx9LGU2ECEfTJ2tSeBwRZ8nSN7ZnydJ5fURqd3McjU56qDkxsxA6wMLVGq3JDEJZ5GzsiP0Ow9CqGei2Cl7X2nuE3xdFO1fanSop9DiTZcHEs3edg1sh2SE7Lws2AwpOaqw8i8eX6Y3h36R6s3XkRTet44scFg3iPhj7ZO7K2wxTpKZptP1t3crUr1T4mftgDKYlS3L0apt42bFI73jtxaPubzaH4T4maycnJUCgUcHfX7Dpn60FBQVrvk5eXx5cCUqkUhjCpoy96NqnN8yZYlxdT8OY5GxCGbRfv8r/ZUEmTKhUwrGUj3Ao3TDLjq8iS52PwuQ2wNjVH83LVML9Bd8RkpfGhkQI3kiN4GUdzawyp7I0ffIZh5IVfkZqv+SF9m40a1hydOtTFnPl/IV+mOn7GZMSolujYqR4+fH87ZPnGV79XbTvelnaDGe7VGR3cmmLeXTakUJhrdS5R1WYwkVlxCM+Mw7aWn6KxUw3cTXuMsqKV6xDUd2jH8yYUgoxvY0MhTIj0Gm6kHOJ/s6ESD+u68HbuiajsAJQVY/r6oUvL2pj11W6NtuPU1WD132HRyTz5c/+qyTwf49ZD7Qm8b6Nhk9uhQ89GmD/+N8jyVe/PGvUqov+YVpg95Gfjm1K6bNkyODg4qBdPz9J15ReXlpUDuUIJF1vNXgkXO2skS1981j2+vTcmdfLhPREhccka+5QpFAhLSNEoH56YigqOpTuD0ZX0vGzIlQq4WGj2SrD15FzNCLYo1o0blZWKIGk8/gi7ghOxj/iMkKJyFDJe5kFaDD6/dwgKQYlBlZtBnyTSbCgUSjgXO7NwcrRGarGz++KGD/bDqGEt8NGi3QiP0H/WeWlIJNn8rMCpWK8LW08t1tNQ3NDhzTFyVEssmPcXz5soUNBDUbxXgu2zeO+FsdFVu8FIZFlQKBVwMtc863Mys0Na3ouDlSGeHTDCqzMW3t+IiCzts3gKxOemID0/ExWtVDMO9CVbIYVSUMBG7Kix3VbsiEz5i2cJtXAZyHsjWE9EYl6kxj4VgpznZxTF1u3NVLOx9CU9Q9X2s1kfRTk5WCOlWM9ncaN6+2BsP1+8v2wfQqML235tYhMlSJNmw8Nd83V806TprO1QwLFIAjrD1tOSXzwDcfD4Nhg2qT0WTdmMiJDCmS0NvKvA0dkG207Nx7/3v+KLeyUnTPmoF/448RHemqDC1dUVpqamSEjQHI9n6+XLq6ZeFbdw4UJIJBL1Eh2t+SYtLfamevQ0Ac1rFjYurKOheQ1P3H/y/A/7hA4+mNalOab/egAPYxJK7PNhdAKqujlrbK/i6oTYNP2eGckENt0zDi3KVVNvE0GE5uWq4n5a6V8z1vVqZvLimQFsrM38JWV0TS5XIvhxvEaSJTt+3k2q4FHg83uERgzxw5hRrTD/0z38/m8rVr+QkDiNJEtWv6bNquDRw+fXb/jIFhg9tjU+nr+TTyktKi4uHSkpmWjWrHCf1tbmqFu34gv3+TZ61bZDV+0GIxcUCMmMQVOnWhqfLZZ0+Uha2B1e3DCvThhdpRsWPdiIkIyXP76rhQPszayR+pJARdeUghxxOaGoWiTJktWQJV0+zdbeg8y0dB2MNm4j8FfkYsTlhpbYZ2zOY7hYaA4DO1tUgkSm3ynbrJ1mU0J963tpfLbYuv/j57f9o/v4YuLAFpizYj+CIl6eQ1bO2RYOtlYvDVR0TS5T4PGjWDRpUUOjjW7SvDoC7z+/x2TIxLYYNb0TPp22BY+LtQen/76LGQN/wszBP6sXNvuD5Vd8MnXz2xNUmJubw9vbG6dPn1ZvUyqVfL1ly5Za72NhYQF7e3uN5XVtPX8HQ5o3RD+fevzaEp8N6gwrczMcvPmQ3750RHfM6dlaXX5iRx+826MlPtt9Ak/TpLxXgy3sPgU2n7vFp6qya1qw6aUjWzdG+3rVsPPKfejbH6FXMKRyM/T3bIxqtq74vHEfWJma40CUqpt1abOBmFO3cEx6cs22aFmuGjysnXj5cdVboa9nYxyOVl2nwsrUDO/X7YxGTh6oYOXAE0G/atIf7pZ2OB6res30ac/+m+jTk117ogG8PF0w993ufEro0RP+/PaF83pjyoR26vIjhzbHxLFt8e3KI4hPkPDcC7ZYWRYePztbS9So5obKXqqzQ08PZ75e2jwNXdq7+wZ691Zde8KrsgvmfNATllZmfDYIs2BRX0ya0kFjuii7BsX3K/5FfLyE90iwhd2nwP49N/DO2NZo2aomqlYrh48X9UVySgYuXSrsui0LXrXt0GW7weyLPodez6494WXtxpMqLU3NcTzuOr99ft1RmFitt7r8cK9OGFe1J74P2on43FTey8EWdh+G/T+lel/Uta8Md0snHqAsaTgJsTnJuJX6/C/yN+V68kF+7YlGjp34tSV6VZwJMxNL3E87xW/vV+kDdHQfpxFQtHcbjcNPVyNdlsB7OdjC7lPgWtJ+PlWV7dfJvAJ8nPuglp0fnzWib38duY1+HRuiV9t6qFLRGfMnduFtx7/nVcMwn8/ogRnD26jLj+nri6lDW+GbjccRlySBs4M1X6wsVJ8t9v/sUe1Qv0YFVHC1h099L3z34QDEJKTh2gP952zt/+MSerJrT/RvCs9q5fh1JSytzHHiwB1++7ylQzBhTjd1+aGT2mHsu12x8rN9SIhN49e2YIulter9mSHJwZPQBI2F9aSyno+YyBf32Oj94ldsSti4cePg4+MDPz8/rFq1CllZWTyj+007dj8ETrZWmN29JVztrPkU0Om/HVAnb1ZwsoOSpQU/M7xlI5iLxTzxsqh1J65i3Ylr/O/TAWH4ct9pTO7ki4UDOiIyMRVzt/6Du5Gx0LdjsQ/hbGGD2XU68YtfsSGNade2ISVPFTmzwIBlBRewNjXDZ436wN3KHnkKGZ9a+vHtfXw/jEIQUNXWFf19m/DrUqTLshGQFouxlzbx6aX6dvZCEL8mxYQxbdQXv5r/6W6erMm4u9lr1K9/n6YwNxfjy88Gauxny/ZL/FoXTOuWNfDxh4VfBosX9S9RRl/OnQ2Eg6M1DxRYcBAWmoCPP9qlTt50Y/VTFtavb/9mvH5ffDVYYz9/bL6IrVsu8r93/nWNNy4fzOsJW1tL+PtHY+FHu8pk3oUh247ziff4NSnGVe0BJ3N7hGU+5T0Q6TLV0KKbhZPGe69PxdYwNxFjcQPN57Y14hi2RR7n7Uw124o8SLEVWyElT4rbacHYEn6E9zrq2yPpRVjHO/BAwYZf/Cocf0V+jiyFKnnTwbwcBBROKfR27gWxiRmGeC3S2M+FxB18YYIzruJI7Dq0LjcU3SpMRUreU+yNWoro7Ed6rh1w6lowHO2tMGVIa7g4WvOponOX70Pqs6Hv8i6an61BXRrD3EyMZXP7aeznt31X8Nu+q1AqBdTwKodebevDzsYCyWmZuO7/BL/svgzZs5w7fbpwzJ9fk2LM7C48OZNNFf102mZ18qZbBUfN9+fw5rzt+GzVOxr72b72NL/WhSGJhKLPtJR+/vlnfPfdd4iPj0eTJk2wZs0afiGb0mAJV2yMtM7spTC10P/0HX0Q2mpmYRubcuv1P21On0xzyt4XdmnJ5bm4cOUrPqTwX8/+X8frth0F7UaHf2ZAbGMBY9S+XNlJ7nwdh5d2hDFzvK+Zm2dM5Io8nA5eWap247Uu0z179my+EELIq6C2gxDjRj8oRgghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJCioIIYQQohMUVBBCCCFEJyioIIQQQohOUFBBCCGEEJ2goIIQQgghOkFBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE5QUEEIIYQQnaCgghBCCCE6QUEFIYQQQnSCggpCCCGE6AQFFYQQQgjRCQoqCCGEEKITYhiISKlajNEpn19hzMbfHwhjpkxNh7ESBBnKMtFiB4hMLWGMzt+ygjGrey0Axux3r0swVtIMJZxqla4s9VQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJCioIIYQQohMUVBBCCCFEJyioIIQQQohOUFBBCCGEEJ2goIIQQgghOkFBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE5QUEEIIYQQnaCgghBCCCE6QUEFIYQQQnRCjDJmROvGGN/RG652NgiOTcKyA2cREJWgtezgFg3Q16ceapZ34euPYhKx+silEuWrujljbp828KnuAVMTE4QnpGDulsOIT8+AvlnbjIeN7QyYmJaDTPYIGemfQia799L7WVr1h6PzeuTmHEN66sRnW8WwtV8AC8tOMDWtDEGQIj/vIjIkS6FUan/N3rS+49thyMwucCpnj/BHT7Huk90IufdEa9nKtSpgzPzeqNnIC+6eLtjw+V4c/PWsRpk/bnzJbyvun83nsXbRbuhb3+ldMXRubziXd0D4gyisnfsHgm+Fay1buW4ljF08BDWbVkX5KuWwft42HPjpWIlyLhWdMPmbEfDt3hgW1haIDUvA91M24vGdCD3UyHj0HeKLoaNbwdnFFuGP47H2+6MIfhSrtWzP/s3QpXcjVKnmxtcfB8Vh87rTzy3/3se90WeQD9avPIYDO6/DEPrN7I6h8/rBubwjwu4/wdr3NiH4ZqjWsj0nd0bXMe1RpYEnX398OxybPvnrueXfXz8FfaZ1w7q5m3Fg9REYQie3TuhZoScczBwQlR2FP5/8iYgs7Z+BduXaobVra1SyqsTXI7MisS9mX4nyFSwrYKjnUNS2qw1TkSlic2Lxc+jPSM1Phd5ZvwORzWTApBwgC4KQ8SUge6C9rNUwiKwGAOJaqnVZAITMlSXLm1aHyO4jwNyPrQCKUAhpswFl3NvTU3HhwgX07dsXFStWhEgkwsGDB6Ev3ZvUwkf922HD8WsYtvJPhMQmY+PUQXC2tdJa3re6B47eCcLEdXsxes1OHiRsnDYIbg426jIeLg7Y+u4wRCSmYeK6PRj8/TZsPHkd+XI59M3Sqh/sHBYjM2MlkhO7Qy57BCfXHTAxKfmlWZSpqQfsHD5Dft41je0ikRXMzBoiK2MVUpK6Iz1lMkzF1eHksgWG0K5fM0z5YhC2/3AEs7svR/ijGHzz12w4uNhqLW9hZYb4JynY9M0hpCZItJZ5r+e3GNlooXpZOGwN337xn7vQt/ZDWmDat+9g+zf7MbP5pwj3j8LSwx/DsZy91vIsQIiPSMSmT3ciJS5NaxlbR2v8eHYx5DIFPun3LaY0mY9fFvyJzPQslCWGbDeY9l3qY9qcbtj+23nMHLsR4Y8TsHTNaDg6WWst39i7Ms4dD8BHM/7AnEm/IylBgmU/jYFLObsSZVt3qIO6DTyQnCiFobQf1grTfhiH7V/uwQzvBQh/8ATLjn3y3Pde4/b1cXbnJXzUaQneb/UJkqJTsPz4p3Cp6FyibOsBfqjbvBaSnxrgi/YZP2c/jPAagUNPD+GLgC8QnR2ND2t/CDtxyePB1LGrg2sp17AiaAW+fvQ1DxLm1Z4HRzNHdZlyFuWwqN4ixOXG8XKfBXyGv2P/hkwpg95Z9oLIbhGEzJ8hJA8A5IEQOW0CTEoeD0Zk3hxCzmEIqWMgpAwDFPEQOW0GTNwLC5l6QeTyFyAPh5A6GkJKXwiZawHkvdGqvHJQkZWVhcaNG2PtWvbk9Gts+2bYdy0AB28+QnhCKr7cewo5MjkG+jXQWv7jP49h15UHvEeDBQ2Ld52EiUiE5jW91GXe69UaFwMj8ePhiwh6moSYFAnOPQxHamaOHmumYm07FdlZO5CTvQsK+WNI0xdAEHJgZT3yBfcygYPTWmRKf4BCrnnGLwgZSEsZgdycf6CQh0EmuwNp+icwM28ME1NVBK9Pg6Z1xrE/r+DkrmuIConHT/N3Ii8nH91HttRaPuR+FH776gDOH7oNWb72IE+Skom0JKl68evaALERSXhw9TH0bfD7PXF001mc2HoBUUFPsXrWJuRl56H7uPZay4fcDsevC//CuT3XIMvTXr9h8/oiKSYFP0z9hfd4xEcm4fYpf8SFJ6IsMWS7wQwe1QJHD97BicP3EBWRjNXLDyMvV4bufZtqLb/88wP4Z98tHnxEP0nBj9/8w4Ohpr5VNcqxIGPmhz2x/PP9kMuVMJTBc/vg6G+ncXzLOUQFxmD19F+Ql52P7hM7aS2/fMwa/LP+BMLuRyI6OBYrp2yAyESEpp0121IWZMxaMxHLRq+GXKb/E60C3cp3w4WkC7iUfAmxubHYGrkV+cp8tC3XVmv5X8J/wdnEszz4iM+Nx+aIzfz41bOvpy4z2GMwHqQ/wJ7oPbznIykvCffS7yFDrv8eapH1RCB7F5CzT9WbIP0cEHIAqyFaywuSD4GcHTz4gCIcgnSR6uvcvLAtFdnOBfLOQ8j8FpA/AhRRQN4ZQJn6dg1/9OzZky/6JjY1QT0Pd/x++qZ6myAA10Ki0LhKhVLtw9JcDLGpKSTZuXxdJALa1a2KzWdvYcPUgahTyQ1PUyX8Mc4EhEG/zGBm1ghZGT8X2Sbw4Qozc+/n3svW7gMolcnIyf4L5ryL68VMTOwhCEoISu1n/m+K2MwUNRt5YtdPx9XbBEHA3YtBqOtdTWeP0WmwH/ZvPKOT/b3qY9dsVhU7v/tbs35nAlC3Rc3X3m/LPt64ffIBPt3xHhq1rYPk2DT8s/EUD17KEkO1G4xYbIKadSpi5x+XNNqOuzfDUbehR6n2YWFpxveTIS082WDtx4IlA7Fn+xU8CU+CoYjNxKjlXQ07lx/QeO/dOfUA9Vo86x5/CQtrc76fjNRM9Tb2Jbxg67vY8/3fePIoBobChiWq2FTBv7H/qrcJEPBI+gg1bGuUah8WJhZ8P1kKVQ+fCCI0cmyEo3FHeY+Hl7UXDyrYY9xN13cvpxlgVh9C1oYi2wQg/wpEZk3ZXy8nsgJEYhZtFGwALDpAyPpN1eMhrgcoYlSPkXcKb1KZSdR0srHigUVKRrbGdrbuYqe9C7O4uX3aIkmSyQMRxtnWGjaW5pjYyReXgyIxbeN+nPEPw4/j+8Knun7P5E1MnCESiaFUajZOCkUyz6/QxszcD1Y2IyBJ/6iUj2IBO/tPkJtzEIJQ2Hjog72zLUzFpkhP0jwLYOtObtq7aF9Vyx6NYWtvxXtC9M3e1Y7XL63YME1aohTO7g6vvd8KVcuhz9TOeBoaj4V9VuDwL6cwc+VYdB2t/QyNlGTvaA1TsQnSUjWHjNg6y68ojcmzuyAlOQN3bhTmxwwf2wYKuRIHdxkmh6KAw3PfexI4lS/s7n+RyStGIyU2FXdO+au3DV/QH0q5AgfWGCaHogAb4mABgVSuObwkkUlgb1a6toPlTaTnp+Oh5KFqn2Z2sDK1Qu8KveGf7o/vg7/HnbQ7mF1zNs+v0CsTJ972Q5msuV2RosqvKAWeN6FIBPIuP9unC0QmthDZTIWQdwFC2gQIeScgclwLmL385POtTtTMy8vjSwGp1DDjjpM6+aJn09qYuHYP8uUKvo0NhTDnHoZh2wVVdMqGSljPx9CWjXAr7CneViKRDRyc1kCS9hGEUnVnieHovJFHsNL0j2GMeoxqiZtnHj03/6IsEpmY8GGSzZ+rkk5ZAl6V+p7oPaUzTm6/CGP1trQbzPCxrdG+awN8NGMLZPmqtqNmnQoYMKI5Zo5hn6mybfiCAegwvDXmdVwMWZ4qn6Bms2oY+F5vzPSej7KuV4Ve8HPxw4rAFZALqiEck2fn06xX4kTCCf43GyphPR8d3DogOCMYZYbNVMCyN8+bAPI1+wvyTgPZz3Lo2FCJWTOIrEdCkNwou0HFsmXLsGTJkv+8n7SsHMgVyhK9Emy9eO9FceM6eGNiZx9MWb8fIXHJGvuUKRQIi0/RKB+RmIqmVfXbU6FUpkIQ5DApFpmamrpCqSjZtWoqrgKx2AtOLn8U2ap6I7lXjEJyQlsoFE80AgpTcSWkJg/Tey8FI03NhEKugGOxRDe2zs7m/ys3D2c0aVsHX036FYYgTc7g9XMq1ivBemH+S5CTGpeOqEDN4Jbla7QZ4Atjpqt2g5GmZ/MeBSfnwgRthq2nprz4szDknZYYPq4NFszeiojQwjyWBk284Ohkgz//nqvexnpDpr7fDQNHtMDYAauhL5LnvvcckBaf/sL7DvmwL0YsGIAFXb9EhL+qB5dp0LYOHN3s8eeT9eptrDdk2vfjMOj93hhTbRb0heU4KAQF7MWavRJsFohU9uK2o0f5Hrw34rvg7xCTE6OxT7lSzmd7FBWXE4eadq8/XPlalGm87YeJq+Z2UxegWM91CdaTILKZBiF1HCAPLrZPGQR5sdk88jDgBcPpZWL4Y+HChZBIJOolOjr6tfbDAopHMQloXlM1BYphHQ0tanrifuTzp8dM6OiDaV2bY8YvB/j9i+/zYVQCqrhpZthWLueEuDR9nxnJIJM9gLlFmyLbRHxdln+7RGm5LBTJCR2RkthVveTlnkB+3mX+t0IRWyygqIrU5OEQlNpnGbxpbPbC4wfRaNKmtsaYLVsPvK19yuWr6Da8BW9cb5wKgMHqdycCTTrW16xfxwYIvPb6SaMPr4bAo5ZmzpBHzQpIiCrWVWpkdNVuMCyB8nFQLJr4VtNoO5r4VEOg//NzBYaOaYV3JrXDove343GgZhtz6ugDTB+1HjNGb1AvbPYHy69Y9N526BNLoGS9WU07N1Rv40mlnRvi0bWQ595v2Ef9MPrTIVjU8xt+/6JObbuAaY3nYXrTj9QLm/3B8isW9vgG+sQCCjYltJ5DYZIly4moa18XoZnap8AyPcv3RN+KffFD8A/8/tr2Wd6yvMZ2d0t3pORrnmS+eTJA9hCiIkmWPCfCvBUE2QvyO2ymQGQ7C0LaJEAeoGWf/hCJNROLIa4CqL8bymhPhYWFBV90Yev5O/hmZHc8jE6Ef1Q8xrRvCitzMxy8oRonY7clSjOx+l/VuNLETj6Y1aMlFmw/iqepUnUvR3aeDDn5qm6+zedu4fsxvXE7PAY3QqPRpk4VtK9XjU8v1bfszF/g4LQKMtl9yPLvwsZ2CkQia+Rk7+S3OzithkIRj0zpMj4tSF40MmWpPSz50gRFtrOA4lc+rTQtZSxEMIXoWU+IUsnOYPQ7dWr/xtOYt3osHt+PQvC9SAyc0gmW1hY4sVOVAzFvzVikxKdj89K/1cmPXs++UNnfruUdUa2+B3Ky8hAXmaTRgHYd0RInd1+HUmG4DPx9q4/io9+n4fHtCATdCsOgd3vA0sYCx7ee57d/9Pt0pMSmYdNnu9R18qqrShQ0MxfDtaITqjWqjNysXH4tCmb/mqNYdX4xRszvhwv7rqO2T3X0mtQRq2b+DmOmy3aD2bfjGj5aPACPA2MR9PApBo1oAUsrMxw/rLoGzEdfDEBKYgY2rTvN14eNbY2xUztg+Wf7kRCXDicXVS9HTnY+cnNkyJDk8KV48JKWkomYKH1/KQH7fjyM+VtmIeRWGIJvhGLgnN6q995mVULv/C2zkRybik2LdvD14fP7Y+yS4Vj2zmo+o8jJXZV7kZOZy99/LGGzaNImr59MjtT4NMSEvNkvJW1OxJ/A5GqTeSAQnhnOZ4Ow5MtLSarkW3Yby5nYG7NXPeQxoNIAbAzbiOT8ZHXuRZ4iD3lK1bDa0fijmFF9Bh/qCJIGoaFDQzRxasKHSfRNyN4EkcO3/HoT7FoTIpvxquRLNhuEtXHsNkUChMwfVHewmQqR7fsQ0j/gCZjqXg4hW7WwP1mSpuMqIP8mkH8NsGgHWHR6NkzyFgUVmZmZCA0tjA4jIiJw7949ODs7w8urcKrmm3D8Xgi/JgULFFztrfkU0Om/HEBKpupFrOBkx7OeCwxr1QjmYjFPvCxq3fGrWH9c9UXGEjO/3Hsakzv74uOBHRGZmIoPtvyDuxH6/+Dk5vzNr0lhZ/fRs4tfPURa8jt8dgdjyqaBCqX/0jQ1LQ9Lq+78b1d3zYzf1KTByM+/Cn268PcdOLjYYcz8PnAqZ4fwh0/x6ai1SE9WJW+6VXKCoCw8fi7uDlh3aqF6nV00iy0ProRg/uDC7uWm7WrD3cMZJ3bqtz7Fnd97DQ7l7DD28yFwYhe/uv8En/RdgfRnwztuni6a9avohA03l6rXh37Qhy/3zz/CR91UZ4PsDHLJsFWY+NVwjP5kIP8CWD9vO87svIKyxJDtBnP+1EM4OFnzQMGJXfwqJB6fvP8n0p8lb7q5O2gcG3YhK3NzMT5fMUxjP9t+PYdtv6qCxLfJ+d1X+DUpxi0ZzpMzw+5F8h6I9ETV0Jubl6tm/aZ3g7mFGRbvnaexn61LdmPbEv2fUL3MjdQbPGGTBQoFF79aGbxSnbzpYu6i0fZ3dOsIMxMznnhZ1MGnB/m1LhiWmMmmpvau2BvvVH4H8TnxWPt4LR5n6n86OnKPQGDJ+nbvP7v4VaCqB0L5LEA1raiaEfIMy4sQicwhcio6WxAQMtdAyPxJtZJ3EoJ0MR8egf1ngDwCQvpsQFay51uXRELRI1EK586dQ8eOHUtsHzduHLZseflFlVjClYODA+rOXApTC0sYo5MffgdjNt5nIIyZMvXF49BlmVyQ4axsDx9SsLfXzawbfbYbHZt9DLGpcbYbwi3DDN3pi8e10s20Kat+9yqcsmxspBlKONUKL1W78co9FR06dNCICAkh5GWo3SDk/0OZuU4FIYQQQt5uFFQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJCioIIYQQohMUVBBCCCFEJyioIIQQQohOUFBBCCGEEJ2goIIQQgghOkFBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE5QUEEIIYQQnaCgghBCCCE6QUEFIYQQQnRCDEMRni1GyM3UBkYtXwZjJsjyYawEoWwfu4iBdjCxtIQxEoa3hDELPwejVgsNYayUubkAPilVWeqpIIQQQohOUFBBCCGEEJ2goIIQQgghOkFBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE5QUEEIIYQQnaCgghBCCCE6QUEFIYQQQnSCggpCCCGE6AQFFYQQQgjRCQoqCCGEEKITFFQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYTohBhlzIjWjTG+kzdc7WwQHJuEZfvPIiAqQWvZwS0aoK9vPdQs78LXH8UkYvW/lzTKfz2yG/r71de436XASMz45QAMwvodiGwmAyblAFkQhIwvAdmDl9/PsjdMHFdByD0JIX2merPI9l1+G0wqAJABsgAImT8CsvswhL6TOmDI7G5wcnNA+MMYrPv4L4TcidRatnLtChizsD9qNvaCu5crNizahYMbT5co51LBEZMWD4JP5wawsDJHbEQSVr67BY/vPYG+9ZvZHUPn9YNzeUeE3X+Cte9tQvDNUK1lK9fzwLglw1HTuxrKV3HDurmbcWD1EY0yYxYPxdjFwzS2RQU9xaR6c95oPYzR6CaNMcXXB+VsbBCYlIQlp8/iQXy81rLDGzbEwPp1UcvVla8HJCTg+4uXNcp/26M7BjfQbDsuRERiwr79MIQxrH4+hfX74syL6zeonmb9vrtUrH7du2NIsfqdZ/Xbb5j6jW7aGFP8ntUvMQlLTr2gfo2eHb9yz+oXn4DvLxSrX8/uGNyw2PELj8SEvYar3+Qi9fvyJfUbUKx+PxSr34rn1G/iG67fKwUVy5Ytw/79+xEUFAQrKyu0atUKK1asQO3ataEP3ZvUwkcD2uGrPafx4Ek8xrRvho3TBqHvsi1IzcwpUd63hgeO3gnCsog45MvlmNjJFxunD8LAFVuRKMlSl7sUGIFP/zqhXpfJFTAIy14Q2S2CIP0cyL8Pkc04iJw2QUjuBihTn38/00oQ2X0MIf9miZsEeSQg/RJQRAMiC4isJ0DktBlCUhdAeME+34B2A3ww5auh+Gnenwi+HYEB0zrjmz3vY3LzzyFJzihR3sLaHPGRSbh46Damfa35xVrA1sEaK4/Mx/1Lwfh0+Bq+n0rV3JGZng19az+sFab9MA5rZvyCwOuhGDSnN5Yd+wQT67yP9CRpifIW1haIi0jEhb1XMX3l+OfuNyIgCgu6fqVeVxjq/fkfGLrt6F27FhZ1aI/PTp3G/bg4TGjWDFuGDELXTZuRkl2y7Wju6YF/goJx5+lZ5CnkmObniz+GDEKPLVuRkJmpLnc+IgLzjx5Xr+crDHNseP3aq+p3j9XPuxn+GDwIXVj9ckrWr8Wz+t2OVdVvuq8vtg4ehO5/aNbvHKvfsbegfnVqYVHH9vjsxLPj59MMW4YNQtffnnP8vDzwT2Aw7pw+izy5HNOa++KPYYPQY1Ox4xde7PgZ6LPVq1j9xvs0w+Zn9UvVUj8/Lw8cLlK/qc19+evRU0v9Fui5fq80/HH+/HnMmjUL165dw8mTJyGTydCtWzdkZRV+Qb9JYzs0w76rATh44xHCE1Lx5Z5TyMmXY2DzBlrLf7z9GHZdfsB7NCIS07B410mYiERoXtNLoxx7oVMystWLNCcPhiCynghk7wJy9gGKUFVwIeQAVkNecC8TiBx+gJC5WhU4FJf7D5B/RXWbPBRCxjKITOwAM/005kUNmtkVx7ZdwskdVxAVHIefPvwTeTn56P5Oa63lQ+4+wW9f7MP5Azchy5dpLTP0/e5IepqGle/+wXs8EqJScOfcI8RFJkHfBs/tg6O/ncbxLecQFRiD1dN/QV52PrpP7KS1fMitMPw6fxvO7boCWZ72+jFKuRJpCenqRZpSMgB72xm67Zjo441d/gHYF/AQoSmp+PTkKeTI5BjSQHvb8cGRo/jz3n1+xh+emoaFx09CJBKhlZdnibYjOTtbvUjzDNN2TPJW1W/vw4cITS2s39CG2us398hRbL9fWL+PTzynfoq3o378+D0ocvyOPzt+z6nfB4efHb/EZ8fv2LP6VdZSv6xs9fK21O+z4y8+fh8Wq9+iY6rvtpZvQf1eqafi2LFjGutbtmyBm5sbbt++jXbt2uFNEpuaoJ6HO34/VXg2LgjAtcdRaFyZde2/nKW5GGITU0iyczW2+9TwwLkvp0Gak4sbj6Px05ErJcq8eWaAWX0IWRuKbBN4QCAya8r+0kpkOxtQpgA5ewFz35c/htVwCEopH1rRJ7GZKR/G2LXqqHqbIAi4ez4QdX2rvfZ+W/RojNtnHuGTTdPQsFVNJMel4/Cmczx40SexmRi1vKth5/IDGvW7c+oB6rWo9Z/2XbFmeeyM2Yj8XBkeXQ3B74t2ICk6GWWJIdsOMxMTNHB3x4brN9Tb2OfpStQTNK1YurbDSiyGmYkp0nNzS/Ro3Jg5HZLcXFyNisbKS5dLlHnTCuq3/oZm/S6z+lV4tfqxehTVwsMDN2ZMh/RZ/X64bKD6lXfHhmvFjt+TVzh+Zi84frOmQ5KXi6tPorHyYtmtn1jL8WP1u16kfj/qoX7/KadCIpHw/52dnfGmOdlY8cCC9SQUxdarujmVah9z+7RFkjQT10Ki1NsuBUXi1INQPE2VwNPFEe/1bo31Uwdi9OqdULKoRV9MnCASiSEoi31ZKFIA8+ra72PmDVgNhZDc78X7tugIkcOPgMgKUCZCSB0PCGnQJ3sXW5iKTZGeqDkMkJ6YAc+apfvgaFOhcjn0mdAe+9efxM4fj6BW0yqYsWwE5DIFTu28Cn1xcLXj9UtLUH0mCqQlSuBZp9Jr7zfo+mN8P2EtooNj4VLBCaM/H4ofL3yJKQ0/QE6mvgNf3dFr22FlBbGJCT9TK4qtVyvl489v3xYJWZm4/CRKI3/i+OPHiJZIUdnRAR+2bYNNgwdhyI6/9Np2PLd+2dmoXsr6LWinqt+lovWLjMTx0MeIkUjh5eiAeW3aYPOgQRj8l57rZ/2sftn/8fhlZuJypJbjl/7s+LVrg01DB2HIdsPUL+U/1i9RS/1OPKsfP37t2uD3oYMw9A3X77WDCqVSiTlz5qB169Zo8JwuRCYvL48vBaTSkmPL+jCpsy96Nq2NiWv3aIwrHbsbov77cVwKQuKScfTTiTwf4/pjLcMJbwuRDUQO30GQfPLyACH/GoSUfoCJM0RWwyByXA0hdciL8zTKCJGJiCdkbvn6IF8P849GlboV0Xt8O70GFW/KzWP31H9H+Ech8Ppj/Bm5nudvHNt0BmVRadqOt6XdYFg+RZ/adTBq126NnILDwcHqv0OSkxGUlIxzUybxfIUrUW9x21HM9IL67X5+/YKf1e/85LJXP5ZP0adOHYzaWax+QVqO37SyWb/edergnWL1+7dY/YKTknF22iTee8F6nd66KaVsfDQgIAA7d+58aYKWg4ODevH01BzzKa20rBzIFUq42FlrbGfrKdIXJ+WN6+CNiZ19MHXjfh40vEhMigSpmdnwcnWEXinTIAhywESVzatm6gIoteQHmHpBJPaEyGkjRO6BfIHlAMCis+pv0yJ5IywvQxEFyO5BkC5i3R+8h0OfpCmZPMHQ0c1eY7ujmx0/m39dqQkSRAXHamyLColHOY83fwZcFEsQZfVzcnfQ2M5muaTFp+vscbIk2YgJiUXFGuVRVpWm7dBVu8Gk5eRArlTC1Uaz7WDrSS/J6Zjs482/dMfv3ce/WF8kWiLhZ5uVHfXbdjy3ftalrJ+vL8bt24egt7V+2c/qZ/0ax8/XG9Ob+2L8nn38S7VU9XMyTP1ctNQv+SX1m+TrzYOK0tYvVQ/1e62gYvbs2Th8+DDOnj0LDw+PF5ZduHAh7+osWKKjXy9CYgHFo5gENK9V2LiIRECLmp64/yTuufeb0MkH07o1x4yNB/AoWvvU06LcHWzhaG2FJKl+EsgKsemeDyEyb1lkmwgwbwVBdrdkcXkYlMm9eA9EwYK804W9Eoq4Fyd3isyhT2w44vH9KDRpV0e9jSVONWlXF4E3w197v4+uh8Kj2BdsperuSIzWby+MXCZHyO1wNO3cUKN+bP3RtcLesP/K0sYSFaqXR2qcfoevdKW0bYeu2g1GplTyKZOtvAoDbRGAll5euBv7/M/JVF8fzG7ZAhP2HYB/wsvbjvK2tnwoIlFPyacvqx9bvxv34vq926IFxu8vA/WLT0CrysWOX+WXHD8/H8xu1QIT9hyAf3zZq1+rl9RvyrP6TdxzgN+/NPVztLJ6aSCm1+EPlnj27rvv4sCBAzh37hyqVq360vtYWFjwRRe2nruDb0Z1x8PoRPjzKaVNYWVuhoPXH/Lb2W2Jkkys/vcyX5/YyQezerbEgm1H8TRVqu7lyM6TISdfxu87o3sLnHrwGMnSbHi6OuCDvm0RlZyOy0H6v8aBkL0JIodv+bUk2LUpRDbjVXkQbDYIe6Ox2xQJEDJ/YGMagPxxsR08mxVQsF1kBZHNDAh5ZwBFoipvw3o0YOoOIbcwYVJf9q87iXlrJ/DhiuA7ERg4rQssrc1xYofqeM1bNwEpcenY/NUBdXKnV21VvoXYXAzXCo6o1sADOVl5iItQ9d4c2HAKK49+jOFze+LCwVuo3awqeo1ti9UfbNN7/fb9eBjzt8ziszqCb4Ri4JzesLSxwPHNZ/nt87fMRnJsKjYt2vGsfmJ+rQrGjNWvkguqN67CcyViw1Tzzad+NwbX/rmNhCdJcKnohLFfDIdSocTZv1SvWVnxqm2HLtsNZtOt2/iuZw/+5Xk/Lp5PubQ2M8PeAFXb8X3PHojPzMT3F1UJvlP9fDGnVUvM/fcoYiQS9VlytkzGF3bf91q1xLGQx7yRZmPyC9q1w5O0dFyM1H/b8fvt2/i+Rw/+5Xk/Pp5PmdWoX48ePKfgu0uq+k3zfVa/Iy+oX8uWOPb47agfP369ntWPHT+fZ/Xzf1a/Xs+O34Uix69NS8w9fBQxUom6Fyc7v0j9WrfEseAi9evwrH4Rhq3fg7h4PqXUqkj92G0Jr1i/d1u3xPFn9fPSY/3Er9ptuWPHDhw6dAh2dnaIf3ahDdY9yeaev2nH74XA2dYKs3q0hKu9NYKeJmH6xgNIyVQNf1RwsuONV4FhrRvBXCzGjxP6auxn3bGrWH/8GpSCErUquqKfbz3YW1kgUZqJq8FR+PnIFcgMMR879wgElvdg9/6zi18FQkibpJrdwZhWfJYXXEqCAhBXh8hqIM+nYEMskPlDSBnJp5fqG/vSZwmNYz7uByc3e4QHxODTYWuQnqQKhtwqOUNQFtbPpbwj1p3/XL0+5N3ufHlwKRjz+/+gnnb65dh1mPDZILwzrw/io5Kx4ZNdOLu3MJNaX87vvgLHcvb8glZO7OJX9yKxqOc3SH82vOPm5apZv4pO2HD3O/X6sHn9+HL/3EPM6/QF38YCjUU73oedix0kSVIEXArCey0XQZJsuByD12HotuPf4BA4W1tjTutW/AuUTaVkFzkqSI6rYG+nkbz2TuNGsBCLsa6/Ztux+spVrLlyFQpBQG1XVwyqXw92FhY8Se5S5BOsvHzFINdy4PWzssbcIvUbv2+/Ormx4vPq169k/VZfVdWvTjlV/eyf1e/ikyf40VD1C1LVb06bVvwLlE2lnLDnBcev6bP6DShWv8tXsebys+P3rH52lkWO30XD1O9IUAhcntWvnI01HiUmYWKR+hU/fqOaqr7b1har35oi9aujpX4/6qF+IqHot/DLCrPxBi02b96M8eOff/GeoljCFWtI6s5YClMLSxij+wvWwZj1qt8RxkyRVjaHFkpDLshwDof4kIK9vWZ+y5v0X9uOgnajylffwMTSONsNQftLZDREZe+aba/EmA+fMjcXEYs/KVW78crDH4QQ8qqo7SDk/wP9oBghhBBCdIKCCkIIIYToBAUVhBBCCNEJCioIIYQQohMUVBBCCCFEJyioIIQQQohOUFBBCCGEEJ2goIIQQgghOkFBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE5QUEEIIYQQnaCgghBCCCE6QUEFIYQQQnSCggpCCCGE6AQFFYQQQgjRCQoqCCGEEKITFFQQQgghRCfE0DNBEPj/ivxcGCtphhLGTC7kw5gpBBmMlRwyjc9hWVHwfJW5xttuCCIYNZECRs2YD5/y2eeuNO2GSNBz6xITEwNPT099PiQhpJjo6Gh4eHigrKB2g5Cy0W7oPahQKpWIjY2FnZ0dRKI3H9tJpVLeGLEXw97eHsaG6le26bt+7OOekZGBihUrwsSk7Ix+UruhW1S/sk36Frcbeh/+YE/IEGdI7IU3xjdXAapf2abP+jk4OKCsoXbjzaD6lW32b2G7UXZOVQghhBDyVqOgghBCCCE6YfRBhYWFBRYvXsz/N0ZUv7LN2OtXVhn7caH6lW0Wb3H99J6oSQghhBDjZPQ9FYQQQgjRDwoqCCGEEKITFFQQQgghRCcoqCCEEEKIThh1ULF27VpUqVIFlpaWaN68OW7cuAFjceHCBfTt25df4YxdYfDgwYMwFsuWLYOvry+/eqKbmxsGDBiA4OBgGIv169ejUaNG6gvXtGzZEkePHjX00yL/B22HMbcbDLUdhme0QcWuXbvwwQcf8Gk3d+7cQePGjdG9e3ckJibCGGRlZfE6scbP2Jw/fx6zZs3CtWvXcPLkSchkMnTr1o3X2RiwK0MuX74ct2/fxq1bt9CpUyf0798fDx8+NPRTI0bedhhzu8FQ2/EWEIyUn5+fMGvWLPW6QqEQKlasKCxbtkwwNuwwHjhwQDBWiYmJvI7nz58XjJWTk5Pw22+/GfppkP+jtsPY2w2G2g79M8qeivz8fB7JdenSReO3A9j61atXDfrcyKuTSCT8f2dnZxgbhUKBnTt38jMp1pVJDIvaDuNCbYf+6f0HxfQhOTmZv+Du7u4a29l6UFCQwZ4Xeb1fp5wzZw5at26NBg0awFj4+/vzhiA3Nxe2trY4cOAA6tWrZ+in9X+P2g7jQW2HYRhlUEGMBxsfDQgIwKVLl2BMateujXv37vEzqb1792LcuHF8PPhtahwIKcuo7TAMowwqXF1dYWpqioSEBI3tbL18+fIGe17k1cyePRuHDx/mGeuG+NnrN8nc3Bw1atTgf3t7e+PmzZtYvXo1Nm7caOin9n+N2g7jQG2H4RhlTgV70dmLffr0aY2uMLb+No09Ee1YDhlrFFi33pkzZ1C1alUYO/b+zMvLM/TT+L9HbUfZRm2H4RllTwXDpoSxbiEfHx/4+flh1apVPKFlwoQJMAaZmZkIDQ1Vr0dERPAuMZaQ5OXlhbLebbljxw4cOnSIzzePj4/n2x0cHGBlZYWybuHChejZsyc/ThkZGbyu586dw/Hjxw391IiRtx3G3G4w1Ha8BQQj9tNPPwleXl6Cubk5nyZ27do1wVicPXuWT5UqvowbN04o67TViy2bN28WjMHEiROFypUr8/dluXLlhM6dOwsnTpww9NMi/wdthzG3Gwy1HYZHP31OCCGEEJ0wypwKQgghhOgfBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE5QUEEIIYQQnTD6oIJdaeyLL754q644pktUv7LN2OtXVhn7caH6lW15b3H9jP46FVKplF9Njf34ir29PYwN1a9sM/b6lVXGflyofmWb9C2un9H3VBBCCCFEPyioIIQQQkjZ/EEx9otqsbGx/MdeRCKRXrqJiv5vbKh+ZZu+68dGO9kPEVWsWBEmJmXnnILaDd2i+pVt0re43dB7TkVMTAw8PT31+ZCEkGKio6Ph4eGBsoLaDULKRruh954KdqbBtGoxH2KxBYxRZF/jrFeBmr+lwNjt/fcQjJE0U4nKzSLVn8OyouD5PrlTBfa2ZaeH5VW0XzoJxsx7/H0Yu4gOb99sDF2QQ4ZLOFKqdkPvQUVB1yULKMRiSxgjE0vjDirEpsZdP8bezji/uAroYwjhTTxfFlAY67ExNTfO9rCAua05jJ1YpIRREkrfbhjnp5MQQgghekdBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE5QUEEIIYQQnaCgghBCCCE6QUEFIYQQQnSCggpCCCGE6AQFFYQQQgjRCQoqCCGEEKITFFQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJMcqY/gO8MWx4czg72yIsLAE/rTmB4KA4rWUrV3HF+AntUKtWeZQv74i1P5/E/n03NcqYmIgwdlxbdOnaAM7ONkhJzsTx4w+wfdtlGMKYRk0wtZkPylnbIDA5CV+cP4P7CfFay46o3xCD6tRDLRdXvu6fmIDvr14qUb66kzM+bt0OfpU8IDYxwePUFMz892/EZmZA3/qOaoEhE9vBydUW4UHxWPfN3wjxj9FatnINN4x5tytq1q8E90pO2LDsMA5u1Twuf5yaz28r7p8dV7H2q7+hd9bvQGQzGTApB8iCIGR8CcgeaC9r0Q0i2+mAaWXVR1HxBELW70DuocIyJi4Q2c0HzFsDJvZA/k0I0i95WfIGj43VMIisBgDiWqp1WQCEzJUly5tWh8juI8Dcj60AilAIabMBpfY26U0a1qExxnb1gYuDDUJikvDtzrN4GKm97RjYpiH6tKiL6hVVbUdgVAJ+PnhZo/wX47qjX6v6Gve78jASs9fshyG0L9cF3dx7wd7MATE50dgVtRWR2eFay7Zx7YDmzm1Q0cqDr0dlR+DQ0z0a5cdVnoqWrm017vdQ8gA/hX4HQ+g3szuGzusH5/KOCLv/BGvf24Tgm6Fay/ac3Bldx7RHlQaefP3x7XBs+uSv55Z/f/0U9JnWDevmbsaB1Ufevp6KtWvXokqVKrC0tETz5s1x48YN6EOHjnUxfUZnbP3jEqZP3YSwsESs+HYEHB2ttZa3tDBDXGw6fvvlHFJSMrWWGTGyJfr1b4af1hzHhHG/4NdfzmL4iBYYOMgH+ta7Zm180rY9Vl+/ij47t/Gg4o/+g+FiZaW1fPNKnvg7JAgj9+/GoD1/IS4zA1sHDIa7ja26jJeDA/YMGYGwtFRerueOP/DzjWvIU8ihb+16NsSUBb2xfe1pzB78M8KD4/DNrxPh4GyjtbyFpTnio1OxaeUxpCZJtZZ5b+hajGz7jXpZOPE3vv3iMX/onWUviOwWQcj8GULyAEAeCJHTJsDEWXt5IR1C5noIKcMgpPSFkLMPIoflgHkbdRGR43rA1BNC2gwIyf0BRSxEzn8AIu3vibedodqOVz02IvPmEHIOQ0gdw48PFPEQOW0GTNwLC5l6QeTyFyAPh5A6WnUMM9cCyIO+dfOphQ+GtMcv/17DqG+243FMEta+NwhOdtrfJ961PHDsZjCmrtyD8Sv+QkJaBta9PwjlHAvbDuZyQAS6frRBvSz87V891ajY83VqjiEeo3A47gCWBn6GmOwovFtzPuzE9lrL17Kti1tpV/FjyFJ8G7QEafmpeK/mfDiaaZ6ABEjuY/792erl9wh2/PSv/bBWmPbDOGz/cg9meC9A+IMnWHbsEziW016/xu3r4+zOS/io0xK83+oTJEWnYPnxT+FSseT7ufUAP9RtXgvJT1P1UJPXCCp27dqFDz74AIsXL8adO3fQuHFjdO/eHYmJiXjThgz1w5F/7+H4sQd48iQZq1YeRV6uHD16NtZaPjg4Dr9sPIOzZx9BJtP+JVq/fiVcuRyC69fCkJAgwYULQbh1KwJ16lSEvk1u6o1dAf7YG/gQoamp+OTMSeTIZRhar6HW8nNPHMF2//s8+AhPS8XHp09AJBKhtaeXusy8lm1w7kkEll++gEdJiYiSSHAqIgwpOTnQt0Hj2uLYnps4eeA2osIS8dMXB5GXm4/uzwngQgJi8Nv3R3H+yAPI8hVay0jSspCWnKle/DrUReyTFDy4GQF9E1lPBLJ3ATn7VGes0s8BIQewGqL9Dvk3gLyTgCIMUEQB2X8A8mCIzJ+9HqZVIDJvqtqP3B9QRKj+hiVg2QdljSHbjlc9NoLkQyBnBw8+oAiHIF2kai7NWxbu03YukHceQua3gPyR6hjmnQGU+mm8i3qnizcOXArA31ceIiIuFd/8eQq5+XL0b9VAa/lPNx3FnvP3eY9GZEIavtx6krcdfnVUZ74F8uUKpEiz1UtGtv4DJqaLe09cTj6HqykXEZcbix1RmyFT5qGVSzut5TdFrsf5pNOIyYlCQl4ctj35DSKRCWrb1dMoJxfkkMol6iVbkQ1DGDy3D47+dhrHt5xDVGAMVk//BXnZ+eg+sZPW8svHrME/608g7H4kooNjsXLKBohMRGjaWfN4syBj1pqJWDZ6NeTP+Q40eFCxcuVKTJkyBRMmTEC9evWwYcMGWFtbY9OmTXiTxGIT1KpVAXduR6q3CQJw504E6tWv9Nr7ffjwKZo2qwIPD1WEV626Gxo28MSNG2HQJzMTEzRwc8el6Cj1NoGdKURHoVmFCqXah5VYzPeTnpvL10UAOlaphoi0NN7jcXPyDBwYNgpdq9WAvonNTFGzfkXcvVrYPScIAu5eDUPdJl46e4xOfZvg+P5b0D8zwKw+hPwrRbYJQP4ViMyalm4X7AvLtCqE/GdDdCLzZ7vJ19wn8gsDjzLEUG2HTo4N6xkSiVm0UbABsOgAQR7JezxE5a5B5LwXsOgCfRObmqCulzuuBz7RaBuvBz1Bo2qlazsszcUQm5pCmqVqOwr41PLAqe+mY/+S8Vg4qjMcbCyhb6YiU3hZV0Gg9KF6mwABgRkPUc22dG2ZuYkF30+2Iktjey3bOvi20Vp8Uf9bjPQaDxtTzZ4afRCbiVHLuxrunHqg0Tay9Xotng2/vYSFtTnfT0ZqYY88CxIXbH0Xe77/G08eaR9iNnhORX5+Pm7fvo2FCxeqt5mYmKBLly64evWq1vvk5eXxpYBUqr0b+2UcHKxhamqCtDTNNwVb9/Rywev6a8cVWFubY/Mf06BUKnl9Nv1+DqdPFb6B9cHJyornOyRna9YvOTub50SUxoLW7ZCQlYVL0arGxcXaGrbm5pju44cfrl7ivRXtK1fFht79MGr/blx/qr83mr2jNUzFpkgvNgyVnpIBz6rldPIYLTvXg62dJe8J0TsTJ4hEYgjKZM3tihTAvPrz7yeyhajcpWcBhBKC9Asg/1neCOtWVzyFyPZDCNLPVGfWNhMgMq0AgeUFlCGv2nboqt34T8emCJ43oUgE8i4X5rqY2AI2UyFk/ghkfAdYtIXIcS0fMoFMT8M6ABxtrXhgkZqheZadKs1GlfKlazveG9QWSZJMXA+M0sifOHP3MWKTpfAo54DZA9rgp3cH8eESJYta9MRWbMcDAtaTUFSGTIrylqXrUR5UaTgksjSNwOSh9AHupt9Ecl4Sylm4Y0CloXi35jysCFrCgxZ9cXC1421jWoJm/dISJfCsU7oT5skrRiMlNhV3ThUO+w5f0B9KuQIH1rzZHIr/FFQkJydDoVDA3b3IuCLA14OCgrTeZ9myZViyZAneVh061EPnLg2w9OtDiIxMQvUa7pg1qwvPwThx3ADj8q9purcf+taqjZH7diNfoRoqMBGxvgrgZHgoNt27w/9mQyXeFSpiVIPGeg0q9KHHYB/cvBiC1CT9J6C+NiELQko/QGTDeypEdgshsG50NjQCOYS0WRA5LIOJ+20IgpyfXQt55571Q5Udr9p2vFXths1UwLI3z5tgvUQanbx5p4HsLaq/2VCJWTOIrEdCkOgvqPivxnf3RXffOpj6w24+3FHgxK1g9d+hscl4/DQZ/3wzCT61PXAjKBplRXf3PvBxboGVIUshF2Tq7bfSrqn/js2NwdOcKHzdcCVq2dVFcMYjlBXDFwxAh+GtMa/jYsjyVPWr2awaBr7XGzO95xvflFJ2ZiKRSNRLdPTrvRklkmwoFEo4OWkm9bH11FTNs/tXMXV6J+z86yrPu4iISMKpkwHYu/cmRo5qBX1Ky8mBXKmEq7Vm/VytrZFUrPeiuClNfTDDxxdjD+5DUEqyxj5lCgVCU1M0yrP1inZ20CdpejYUcgUcXTS7Fx1d7JCW/N+DALeKjmjSsgaO7dWc3aM3yjTVl76JKptezdQFUCa94I6CaiyefSFlbwJyj0FkM73wZvlDHnQoE5pCSGwNIW0SIHIC5GWnUTdku/Hfjg2bMTIJIptpEFIn8HwXzX3KIMiLZdvLwwDT0g056Ep6Zg7kCiWc7TQT1p3trZEieXHbMaarNyb08MXM1ft40PAiT5MlSMvIhmc5R+hTpjwDCkEBe7GDxnY7M3tIZekvvG9X917oXr4PVj/+Fk9zXvweSs5P4r0fbhaage+bJknO4G2jk7tm/ZzcHJAW/+L6DfmwL0YsGICF3b9ChH9hL1ODtnXg6GaPP5+sx7H8nXwpX8UN074fh23ha9+eoMLV1RWmpqZISEjQ2M7Wy5cvr/U+FhYWsLe311heh1yuREhIHM9/KMBOxNn6o4dP8bosLcRQKjW7uvgwiJ5PBGVKJQISEzSSLNlTaOXphTtxz5+eNq2ZL2b7tcC4Q/v5lNLi+3yQmIBqxYZPqjo54WnGf+hOfg1ymQKPH8aiSYvqGmN+bD3wXuGH4XV1G+gNSWombpwv0vDrlQyQPYSoSCIfP4LmrSDI7r7CfkwKcymKEjIBIVU1/dSsAQR2hlyGvGrboat24z8dG5spENnOUgVy8gAt+/SHSFxVc7O4Cp+ho08soGBTQv3qFmk7RIBfHS88CH9+2zGumw8m926B2WsOIPCJ5nHRxs3RFg42Vkh6SaCiayygiMqORB37wiRLEUSoY1cf4Znap1Ay3dx7o1eF/nyKKJtS+jJsZoiN2BaSlwQquiaXyRFyOxxNOzfUaBvZ+qNrIc+937CP+mH0p0OwqOc3/P5Fndp2AdMaz8P0ph+pFzb7g+VXLOzxzdsTVJibm8Pb2xunT5/W+AJm6y1bFv3Avhl799xA7z5N0K17Q3h5uWDO3J6wtDTjs0GYBQv7YtLkDhrJndWru/FFLDaFq6sd/7tixcJpRVevhuKd0a3QvEV1uLs7oHWbWhgytDkuXXr+wXxTfrt7W33tCZZH8XXHLrAWm2HvI1WD9kPXHvioVeF0w2nevpjbshUWnDqOGKmE92qwxdrMTF3ml9s3+VRVtt/KDo4Y26gJOletju0P7uu9fvv/uIieQ33RpX8zeFYrh3cX94ellTlOPMuBmLd8KCbM7a6ReFmtTgW+sL9d3ez53xWK5dCwD2DXQd44efAOlAolDEVgPQ3WwwHLgarrF9h/qUrwYzMO2PN0+JbnR6jZTFNdf8LUk5cHm6Fg1R9CTpHrVFj0UF0DgZWx6AyR8xYg7xSQfwlliaHbjlc/NlMhsp0DQbIQUMSoejnYIirsDRCyfuNTVdk1Ldj0UliPBiw6Qcj+E/r256nbz649UQ9Vyztj0agusDI347NBmC/H9+A5EQXGdffFjH6tsOSPE4hNkcDF3povVhaqtoP9P2dwOzSsWgEVXOz5rJAfZ/ZHdFI6rj7S/zVSTiUc5deeaOHchudRsKRKlnx5JeUCv318lWkYUHGYRkDRt+JgbI38FSl5ybyXgy0WJhb8dvb/oEojUNWmOlzMXfmskBnV5yIpLwGPpPof9t7342H0YteeGNseXnUq4b31U2BpY4Hjm8/y2+dvmY2JS0epyw+f3x/jvhyB7yetQ3xkEpzcHfli+SyRliVsRj6M1lhY8JIan4aYkNi36+JXbErYuHHj4OPjAz8/P6xatQpZWVk8o/tNO3c2kCdsjh/fDk7ONvziVx8v2KVO3nRzs4dQpNfBxcUOv/w2Wb3Orj/Blnv3nuDDuaoPPrt41oSJ7fD++z3g6GTNL351+J+72Lb1IvTt38fB/JoUH7RoDVcbawQmJWH8oX1IzlElYFW0s9dIkBrdsDEsTMVY37ufxn5WXb/Cr3XBnAgPxadnT2GGjx8Wt++I8LQ0zDzyN27FvX7vzuu6cNQfDk62GPNeFzi52iE8MA6fTt2sTt50q+CoefzK2WHdgffU60MmtePLgxvhmD/uV/X2pi1rwL2iE07sN0CCZlG5RyCYOENk9/6zCywFqs5ylc+Gn0xZUllh/UTsC8r+C8C0PCDkqhIzJfP4ftRM3SCyWcQTA3lXfc7BZ9dCKHsM2Xa88rGxHgmRyBwip581diNkroGQ+ZNqJe8kBOliPjwC+88AeQSE9NmATP/vwxO3QuBka80DBRYcBMck8YtUFSRvlne202g7hrZrBHMzMb6f3ldjPxv/uYqNh6/y3tualVx5kGJnbYGk9ExcC3yCdYeuQFYk70Jfbqddh53YjgcKqotfReGnx98hQ67qcXU2d+EzJgq0L9cZZiZmmFb9fY39HI7dz691oRSUqGTliRYubWFtas2TOB9JA/B37F4+zVTfzu++wq9JMW7JcDixi1/di+Q9EOmJquRNNy9Xjbaxz/RuMLcww+K98zT2s3XJbmxbsgeGJBKKHolS+vnnn/Hdd98hPj4eTZo0wZo1a/iFbEqDZXE7ODigXZvPIBbrf3qSPoQPVEXDxqr2+hePvRqDI2f3whhJM5RwqhXO8xT+25DC63ndtqOg3UgLqQZ7O+P8dQGfxTNgzPymvsowYNkU5qs5JddYsATXczhUqnbjtS7TPXv2bL4QQsiroLaDEONmnCE/IYQQQvSOggpCCCGE6AQFFYQQQgjRCQoqCCGEEKITFFQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJCioIIYQQohMUVBBCCCFEJyioIIQQQohOUFBBCCGEEJ2goIIQQgghOkFBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnxDAQ2bx0CDYWMEZuW8rDmPU+cB3GrnvFJjBGckEGIBxlVZ8ZYyEWW8IYKWalwZhd2doMxu6zkO0wRtkZCpwr5eGjngpCCCGE6AQFFYQQQgjRCQoqCCGEEKITFFQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJCioIIYQQohMUVBBCCCFEJyioIIQQQohOUFBBCCGEEJ2goIIQQgghOkFBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE5QUEEIIYQQnRCjjBno0QojKreHs7kdwjLjsDr4IAKl0VrLtivXAKOrdkIlK1eITUwRk52MXU/O40T8HY0y/T1aopZdJTiY22DitR8RmhkLQxnSpQne6e0DFwcbPI5Kwg9bz+BReLzWsv07NESvtvVQzcOVrwdFJGD97ksa5Tv41MCgzo1Rp4o7HOysMHrRVr5fQ2no2B/NXIbD2tQZyXlhuJDwExJyg7SWrW7bFt4uo+BoXgkmIlOk5z/F3dQ9CJaeVJfpUmE+6jr00Ljfk8wb+DvmYxhCv5ndMXRePziXd0TY/SdY+94mBN8M1Vq2cj0PjFsyHDW9q6F8FTesm7sZB1Yfee6+hy8YgMnL3sH+1f9i/dwtb7AWxmlA36YYMaQ5nJ1sEBqeiDXrTiEoJE5r2d49GqN7l/qoWrkcXw8Jjcevmy9olP/4w17o0bWhxv1u3ArH/E/3wBCGV/HDuOpt4GphixBpPJYH/IuA9Kdayw7y8kZfjyaoYefO1x9JYvFT0EmN8l82GYj+ns007nc58TFmXt8KQxjepjHGd/KGq70NQp4mYdm+swiIStBadnDLBujrWw81Krjw9UfRiVhz+JJG+a9GdUP/5vU17nc5MBIzNhyAIVR3GIrajmNhaeqC9PzHuJv0LdLyHmotW9V+ICrb9YaDeXW+npYXCP+UtSXK25lVQSPX91DO0hsikSmk+eG4Ej8fOXLt3ykGCSouXLiA7777Drdv30ZcXBwOHDiAAQMGQB86uTfGrFp98UPgPjySRmGoZ1t833Qy3rnyLdJlWSXKS+XZ2BZxBlFZiZAJCrRyrYuP6w1DWn4mbqaG8DKWpuZ4kB6BMwn3saDeUBhSl+a18f477bFi8yk8DI3DiB7eWL1gMIZ9tAlp0pwS5ZvV9cSJq0F4EBKLfJkCY/v6Ys2CwRj58R9ISsvkZawszHA/+ClOXQ/BJ5O7wZBq2nVAW7cZOJuwCvE5gWjiPBj9PFdge/g45CjSS5TPVUpxK+VPpOVHQSHIUdW2BQ8ichRpiMq6pS73JPM6TsV9q15XCDIYQvthrTDth3FYM+MXBF4PxaA5vbHs2CeYWOd9pCdJS5S3sLZAXEQiLuy9iukrx79w37V8qqP31K4Iux+JssiQ7QbTsV0dzJzSCSt/OoHA4FgMGeCD774ZhjGTf0W6JLtE+SaNPHH6XCAePjqF/Hw5Rg5rge+XDsP4ab8jOUX12WKu3wzHipWFgWC+TA5D6F6xAebV64mv/f+Gf1oM3qnWEuubj0P/s6uRml+ybfRxqYqjT/1xP+1f5CnlmFi9Lda3GIfB535CYm6GutylxBB8fq/wSzZfaaD6Na2Fjwa2w1e7T8M/Mh6jOzTDhhmD0O+bLUjNLNk2+tTwwNE7QbgXEYc8mRwTu/jy8oOWb0WipPD1uPQoAp/tOKFez5crYAgetl3R2PUD3ElcipTcANRyHIV2FX/GsahByFOklShfzsob0RnHcTf3PpRCPmo7jUO7imtxPGoochWqk0YbsQc6evyOCOkhPEzZCJkyC/bm1aAU8t6u4Y+srCw0btwYa9euhb4N82qHw0+v42jcLTzJSsQPQfuRq5Chd0U/reXvpYXjYlIAnmQnIjYnBXujLyE8Mw6NHKuqy7Beiz8iTuF26mMY2sie3jh01h+HLzxERGwqlm8+idw8Gfq21zwbKrB4/RHsO3Wf9zw8iUvFN7+egImJCD71vdRljl4OxO8Hr+FmwBMYWhPnoXgoOYJAyTGk5T/B2fgfIVfmoZ5DT63ln2bfR3jmJR5USGWxuJ+2H8l54ahgpfl6sCAiW5GmXvKUhY2+Pg2e2wdHfzuN41vOISowBqun/4K87Hx0n9hJa/mQW2H4df42nNt1BbK85wdCljaWWLj9Pfw4dQMy00p+QZQFhmw3mKGDfPHvsfs4dtIfT6JSsPKn4/yz1au79s/WN98exqHDd3mPRlRMKr5bdRQikQjNmlTWKCeTyZGalqVeMjPfbIP9PGOqtcL+qFs4FH0X4ZlJ+PrBP7xtHOCl2dNQYNHdvdj95AaCpfGIzEzGF/cPwgQi+LmqznwL5CsVSMnLVC8ZslwYwtgOzbDvSgAOXX+E8IRUfLX7FHLy5RjQooHW8gu3HcOuSw8Q/DQJkYlp+OKvk7xtbF6rsG0sCCJSMrLVS0aOYY5fLcfRiJAcQGTGP8iQReB20lIohFxUseuvtfyNhE8RJt0DSX4IMmSRuJX4FX9/ulsXfhc2cJmJ+KzL8E9Zg/T8YGTJYxCXfUFrkGLQnoqePXvyRd/EIlM+RLE98ox6mwCBBwP1HSsDpfjObOZUA542btgQ+vwuZkMRm5qgTlV3/PHPDfU2QQBuPoxCwxoVSrUPSwsxTE1NIM00zAf/RUwghptlLdxO2VFkq4Do7Nsob1WvVPvwsG4KJ3MPXMl+oLG9knUTTKqxD3mKTMRk38W1pE28l0OfxGZi1PKuhp3LC8/qBEHAnVMPUK9Frf+073d/noTrR+7g7ml/vPPJYJRFhmo3GLHYBLVrlseOXdc0Plu370aiXt1KpdqHhYUZ309GhuZnq0kjLxzYORsZmbm4ey8Kv/9xAdJiZfTRNtZ1qIjfQy9qtI3XksPQyMmzVPuwNDXjQ8TSfM1eGx+XKjjbbQGkslzcSA7Hz0GnIJGV7Bl4021jXU93/Hbqpsbxux4ShcZVStk2mot5/STZuSV6NM59PQ3S7FzceByNn/69UqLMmyaCGE4WdRCUtrnIVgEJ2TfgYqk96C1OLLLkbWy+oqDdE6GCTRsEp21F24o/w9G8NrLksfwxYrPOvZF6qJ/LG907gLy8PL4UkEpfr7F3MLPhbwo2dFFUan4mvGzcnns/G1NL7Gv7KcxNxFAISvwYfAC33oJeieIc7az4hye1SNcckyrJRuUKzqXax6wR7ZCcloWbDw3fK1GcldiB50VkyzWjZLbuZK159lCUuYkNJtTYDVORGQRBiXMJq3ggUuBJ5k2EZVyCVBYHB7OKaFluEvp5LseeJ7MhQAl9cXC1g6nYFGkJEo3taYkSeNYp3ReXNh2Gt0LNZtUwy88wOSKGoqt2g3Gwt+bBdmq65mcrLT0bXp6qMfeXmTaxPR/2YIFIgRu3InDhcgji4tNRqYITJo9vhxVfD8WsuduhVArQFydza942sp6Eoth6VVtVvtXLzKnXDUm5GbiWHK7ediUxFKfjAvE0Ow2eNs54t04XrGs+FmMu/QIl9Fg/G1XbyHoSimLrVd2cSrWPuf3aIkmaiWvBURr5E6cfhOJpigQero54r09rrJs+EGN+3Akli1r0xMLUESYiMXIVKRrb2bqdeZVS7aOh63vIUSQjIef6s306w8zEBnWcxiMgZR0eJK9BeetWaFX+O5x7Og3JuYV5hWUuqFi2bBmWLFkCQ8lW5GHS9R9hZWoBb+camFWzLx8KYUMjxmRsXz90bVEbM7/ZzfMrjEW+Mhs7I6bAzMQKnjbN0NZtJg8g2NAI8zjjrLpsSl4EHx4ZV/1PVLJuzHstyrJyHi6YuWoCFnT76oXDI8bI0O1GUaOGNUenDnUxZ/5fGp+tM+cD1X9HRCYjLCIRf22Zznsv7tx7+wL755lYoy16VGyISVc2aeRMHIv1V/8dmpHAkz+PdP4APq5Vea9FWcHyKXo0rY2JP+/RyJk4dleVV8c8jktBSGwyjn4+Eb41PXA9RHvy/9uotuN4eNl2w7mnU3l+BSOCiP8fm3UejyWq3mE2VOJq1QjVHQa/0aDijU8pXbhwISQSiXqJjn69gyWRZUGuVMDJ3FZju7O5LVLzCxOLimPdgE9zUviMjl1RF3A+8QFGV9E+xm1I6Rk5kCuUcHaw0dju7GBdoveiuHd6+WBsH1+8t2IfQqOT8TbKkUugFBSwFmueWbD1bHnqC+4pQCKL5TNF2MyP0Izz8HYe9dzSLODIkafDwfz1ewdehyQ5Awq5Ak7uDhrbndwckBZfMgm1NNisECd3R6y//S2O5e/kS+MO9THg3Z78bxMT450Rrqt2g5FIs6Fgny1Hzc+Wk6M1z4N4keGD/TBqWAt8tGg3wiNePGsqLl6C9PRsVKroCH1Ky8/mbaOLhWbbyNaTi/VeFDe2WmtMqNEW06/9gccZ2mdSFGA9Fql5WfCyKV3Pqa6kZanaRhc7a43tbD25WO9FceM6emNiZx9MW78fj2Nf3DayHovUzGx4uur3+OUp0qEU5HzWR1FsPVf+4udcy3EM7424EDsLkvzQEvtksz2KkuZHwFpcHm/SG2+VLCwsYG9vr7G8DrmgQEjGU97bUIBFY82ca+BheunPClgyi5nJ2zeTln1o2JRQ3yJJliIR+Lp/qPZpb8zo3r6YOKAF5ny7n9//baWEHIm5IfCwKZo4JoKndTPE5zwq9X5EMIGpidlzb7cRu8LS1P4lgYruyWVyhNwOR9PODTXea2z90bXCM6JXwXIopjT8ANObfqRe2PTUM39e4n8rlfob3tE3XbUbjFyuRPDjeI0kS/bZ8m5SBY8CtU+5ZEYM8cOYUa34FFF2/5cp52oHe3srpKTqN5mWtY2Bklg0d62m0Tay9Qdpzw/Gxldvg6m1OmDmta18SunLuFnaw9HcCkm5mXpvGwOjE9C8lqfG8WPr9yOf3zZO6OSDqd2bY+aGA3gU/fK20d3BFo7WVkiW6vf4CZAjLS8Ibla+RbaK4Gbti5Tcwt6i4tj003pOk3ExdjafUlp8n6m5D2FnpplYzNaz3+B0Uubt+3Z9gd1RF7Cw3nAES2MQKInGUK+2sDI1x5E4VQLPovojkJwrwS9hR/n6O1U68rKsp8JcJEYL1zroXt6bzxopYCe2grulE1wtVI2Wl41qXjrr/XhRD8ib8NfR2/h8Wg8ERsTjUVg8RvRoBksLMxw+H8BvXzytB58qum73Jb4+po8vpg5uhc/XHUFssoT3ajA5uTLkPOsut7exhLuLHco5qc5iCvIzUiRZPF9Dn+6l7kGXCh8jMSeYX5uiidNgiE0s8UhyjN/etcLHyJQn42rSb3zd23kkD0RYTwXLqahi2xy1HbriXPwqfruZyBJ+ruMQmnEB2YpUnlPR2m0a0mVP8SSrMKlLX/b9eBjzt8ziszqCb4Ri4JzesLSxwPHNqiGa+VtmIzk2FZsW7VAnd7JrVfC6mIvhWskF1RtXQU5mLmLD4vn/kQ81vxRys/IgTc0osZ282J79N7FwXm8eHAQGx2HIQB9YWprh6AlVo81uS07J4NeiYEYObY4JY9rg6xX/ID5Bwq9tweTk5PPPl5WlGcaNbo0Ll0KQmpaJihWcMG1SBzyNTcPN2xF6r9+28Cv4qskgPEx/yq81MbpaS942HoxSdXN/3WQwEnOlWBOkusbLhOptMbN2J3x8dw9ic9LVvRzZ8nzkKPL5fafX6ohTcQ95boaHjTPm1u2G6KxUXEnSf07a1nN38PU73fEoKhH+UfEY3b4prMzNcPC66roM37zTHQmSTKw5fFlVv84+mNWrJT7eehRPU6XqXo7sPBly8mX8vjN6tMCp+495b4enqwPPu4hKTsflQP0PXYWkb4ef2xIeHKTmBqCm4yiIRVaIzPib3+7rtgQ5iiQEpPzM12s7jkN9l+m4Hv8JsuRxsHjWyyFXZkMhqBJpg9O3oWX5ZUjKvYvEnJs8p6KCTVueU/FWBRWZmZkIDS3sZomIiMC9e/fg7OwML6/nJ9zpAruWhKOZDSZW6w5nCzuEZsRi3t3f1Mmb7paOPOO+APtgfFBnIMpZOCJPKePXq/j64V98PwVal6uPRfWHq9e/aDia/785/AQ2hxdeZEkfTl0PhqO9FaYObg0XB2uEPEnCnG/3IVWq+vJ3d7XXSCBiF7UyNxNj+fv9NPbz6/4r+G3/Vf5322bVeaBS4Jt3+5Qooy+PM87BytQRzctNgI2pE5LywvB39AJ+3QnG1sxNI7mS5VF0KP8+bMXlIBfykJYXjZOxS/l+GCWUcLGohjoO3WBhaosseQq/fsW1pM1QGuBaFed3X4FjOXt+QSsndvGre5FY1PMbpCeqkjfdvFwhFEngc6nohA13v1OvD5vXjy/3zz3EvE5fwJgYst1gzl4IgqODNQ8UCi5+Nf/T3TxZk3F3s9doO/r3aQpzczG+/Gygxn62bL+ELdsvQ6EUUK2qG7p3aQBbG0ukpGbyYGLT1ouQGSCn6XhsAJzMbTCzdmd+8atgaRy/SFXBNSrKWznwz0uBoVV8YW4qxkqfkRr7WR98BhtCzkIpKFHL3h39PJvAzsySX7vialIo1gadhkxpgPrdDYGTrRVm9moJV3trBMck8YtUpT4b/ijvZKfRNg5r3QjmYjFWTuyrWb+jV7H+2DVev5oVXdHPrx7srCyQKMnE1eAo/HzkCmQK/dcvJvMkLEydUN95OizFLkjPC8HF2HeRp1D1uFqbsSGLwvpVdxgCU5E5WlUobD+Yh6kb8Sj1F/53bNZZ3E5cijpOE9DUdR4yZE9wNX4+UnLvvdG6iISin6RSOHfuHDp27Fhi+7hx47Bly8uv8seyuB0cHNDy4LsQ21jAGOVtebNjVoY2+tN/Yez+rle6WQFljVyQ4RwO8TyF/zKk8Kp01W607vwFxGJLGKO0WYa5voq+CKf0m4thCJ+9ux3GKDtDgfHN7peq3XjlnooOHTpoRPSEEPIy1G4Q8v/BeNPHCSGEEKJXFFQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJCioIIYQQohMUVBBCCCFEJyioIIQQQohOUFBBCCGEEJ2goIIQQgghOkFBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE5QUEEIIYQQnaCgghBCCCE6IYaByLe4AWaWMEaJrWHU9s/uBmM39NExGKOcTDnO+aLMyqxgBlNzMxijrBxzGDMzOxi9Y2kNYYzyM/MB3C9VWeqpIIQQQohOUFBBCCGEEJ2goIIQQgghOkFBBSGEEEJ0goIKQgghhOgEBRWEEEII0QkKKgghhBCiExRUEEIIIUQnKKgghBBCiE5QUEEIIYQQnaCgghBCCCE6QUEFIYQQQnSCggpCCCGE6AQFFYQQQgjRCQoqCCGEEKITFFQQQgghRCcoqCCEEEKITlBQQQghhBCdoKCCEEIIITpBQQUhhBBCdIKCCkIIIYToBAUVhBBCCNEJMcqYQd2b4J2+vnB2tEHokySs3HQagWHxWsv269wQPdrVRzVPV74eHJ6ADX9dLFF+8rDWvKydjQUeBMXiu99OIiY+HYYwplETTPX2QTlrGwQmJ+GLc2dwP0F7/UbUb4hBdeuhlouqfv6JCfj+yqUS5as7OePjNu3gV8kDYhMTPE5Nwcx//0ZsRgb0bUC/Zhg+rDmcnW0QFpaINT+fRFBwnNayVSq7YsL4tqhVszzKl3fAz+tOYd/+WxplrKzMMXF8W7RpUwtOjtZ4HJrAywUHa3/N3rQGjgPQxGUErE2dkZIXiosJa5CYG6S1bDXbtmjmMhoO5pVgIjKFJP8p7qXuQoj0pLpMpwofo45DD437RWXewOGY+W+8LsZmaMfGGNvDBy4ONngcnYRvd5zFwwjt75OB7Rqid8u6qF5J9dkKfJKAtfsva5T/YmJ39G1dX+N+V/wj8e6q/TCE0TV8MLl2S5SztEVgegK+vHsMD1JjtZYdXq0pBlRuhFoO5fh6QFocfvA/q1F+hW8/DK7aWON+F+JCMfHiXzCEUc0bY2Ibb7ja2iAoPgnfHD4L/6cJWssO9WmAfk3qoaa7C19/FJuIH09c0ii/dFA3DGymefwuhkRi6tYDMISObp3Ro3xPOJg5IDo7CjuitiMiK0Jr2Xau7dHStRUqWXnw9SdZkdj/dK9G+YlVJ6O1axuN+/lL/LEq5Ie3J6hYtmwZ9u/fj6CgIFhZWaFVq1ZYsWIFateuDX3o3LI23hvbAd/9egoPH8dheO9m+PGTIRg5ZxPSpNklyjet54lTl4PgH/wU+TIFRvf3w6pPh+CdD7YgOS2Tl2HbhvZsiq/XHkVsogRTh7fh+3zng838PvrUu2ZtfNK2PT49ewr34uMwsYk3/hgwGJ23bkJKTk6J8s09PPF3SBBux8YiT6HAdB9fbB04GN22/YGELFX9vBwcsGfoCOx+GIAfr11BZn4eajm7Ik8uh7517FAHM6Z3wo+rjyMwMBZDBvvi2+XDMXbCL0hPL3n8LCzFiI1Lx7nzQZg1o7PWfX70YU9UreKKZcsPIzklA127NMD3347AhIm/ITlF9RroSw27jmjtNhPnE1YiIScQjZyHoI/nd/grfAxyFCWD1FxlBm6nbEN6fhQUghxVbFvyIIKVjc66qS73JPM6zsStUK8rhXyUNYZuO7r61sIHw9tj6bbTCAiPw6iuzfDz3EEY9MlmpGWU/Gx51/bA8RvBuB96FvkyOcb19MXaDwZh6GdbkZRe+L667B+BJZuOq9fz5fptMwr08qyHRY274rPbR3A/9SnG12yOze1GoevRdUjNK/nZ8itXGYejAnAnJQZ5Cjmm1mmFLe3eQc/jG5CQU3iycT4uFAtu/q1ez1cYpn49G9TCgp7t8MXfp/EgOh5jWzXDr+MHodeqLUjNKnn8fKt64MiDINyNiuNt3eR2vvht/CD0XbMViRlZ6nIXQiLwyf4TBj9+vs5+GO45Atue/IHwzHB0de+GubXm4RP/j5EhL3nyV9u+Dm6kXEdo5p+QCTL0LN8LH9T6CJ8FLEK6rLCt8U9/gE0Rv6vX5YLs7Rr+OH/+PGbNmoVr167h5MmTkMlk6NatG7KyCg/SmzSijw/+Pu2Pf88FIPJpCr799STy8mXo07GB1vJLfjqC/Sfu4fGTJDyJTcWyDcdhIhLBp6GXusywXs2wZf81XLwVhrCoZHz58xG4OtminW8N6NvkZt7Y9dAfex89RGhqKj45cxI5chmG1m+otfzc40ew/cF93qMRnpaKj0+dgAgitPYsrN+8lm1wLjICyy9fwKOkRERJJDgVEaY1SHnThg72w79H7uPYcX88iUrBylXHkJsnQ88ejbSWZ70NG385i7PnAiHTEuCZm4vRrm1tbPz1HB74RyM2Nh1/bL2E2Kfp6NevKfStsfNQPJL8iyDJMaTlP8H5+JWQK3NRx6GX1vKx2fcQkXkJaflRkMpi8SBtH1LywlDBSvN4KwQZchSp6iVPqd9gSRcM3XaM7uaNAxcC8M/lh4iIS8XSbaeQmy9H/zba245Pfz2KPWfvIyQ6CZHxafhqy0mIRCL41fXUKCeTK5AizVYvGdl5MISJtVpgV/hd7Iu8j1BpMj67/a+q7ajaRGv5D68fxJ9ht3mPRnhGChbdOszbxpZuVTXK5SsVSM7NUi9SWS4MYVzrZthzKwAH7jxCWFIqvvj7FHJlcgzy1n785u85hr9uPOA9GhHJafjswElV/aoXto0FQURyZrZ6keYa5vh1c++OC0nncTn5EuJyY3lwka/MRxvXdlrL/xq+EWeTziA6JwrxuXHYErmJvz/r2tfTKCcX5JDKJeolW1EywDRoT8WxY8c01rds2QI3Nzfcvn0b7dppr7yuiE1NULuaO7YdvK7eJgjATf8oNKhVsVT7sLQQQyw2gTRT9cGo6ObAA4hbD56oy2Tl5ONRaBzf56krwdAXMxMTNHBzx7qbN9TbBHYmFBWFZuUrlGofVmIxzExNkJ6nqp+I9Q5UrYZfbt/kPR71yrkhRirhj3EyPBT6xF73WrXK48+/rmocvzt3IlG/XqXX2qepqQlf8vM1e11YoNmwgWbj/6aZQIxylrVxJ2VHka0CYrJvo7yV5gf9eSpZN4OjuSeuZm8str0Jxtc4gDxFBp5m38X1pN+Rp5SiLDF021Gnsjs2Hyny2RKAG4+eoGH1CqVvO0xNIc3KLdGjcfLH6ZBm5+JWYDTWHbgMSbEyemk7nCpgQ+BljbbjSmIEmrqousdfxsrUDGKRCST5micbzctVxvV+H0CSn4uriRH4MeAc0ouVedNYm1a/ojt+vXBT4/hdDYtCE89SHj8z1fGT5GgeG7+qHrj08TRIc3JxPTwaq09dQXqxMm+aqcgUlW2q4Ejcv+ptAgQ8kj5EddvqpdqHhYkF30+WXDNIr21XBz82WYNseRYCMwJxIGYfshRZb29OhUQi4f87Ozs/t0xeXh5fCkilr9cYOtpb8cYhNV3zBWHrlSs+//GLmvlOeySnZuGWvyqIYHkZfB8SzeiNrRfcpi9OVlY83yE5W7N+ydnZqP6C17eoBW3aISEzC5eiVPVzsbaGrbk5pvv44Yerl7D80gW0r1IVG/r0w6h9u3H9aQz0xcHBmgcAaWma9WPrXp6qcc9XlZOTj4CHMRgzujXv+WD76tSxHurVrYSnsWnQJ0uxA8+LyJanaj5HeRqcrDXPjooyN7HBuBp7YSIygyAocSHhRx6IFM2fCM+4AKksDg5mldC83GT08VyB/U9mQYASZdXL2g5dtRuMo52q7WA9CUWx9SoVSvfZem9IWySnZ+L6oyj1tisBkThz+zFik6XwcHPArEFtsGbOIExY+heU7FtPT5zMrXnbkZKn2YPFehaq2alyQl5mfqPOSMzNwOWEcPW2C/FhOPE0CNFZ6fCyccK8hh3xe9uRGHpms17r52j97PhlFjt+mdmo6upUqn3M694WiRmZuBJWePwuPY7EyUehiEmTwMvZEXO6tsbGcQMxcuNOvdbPTmzHAwKpTPWZKCCVSVHBsnRB0xDPoUjPT8cj6SP1tgCJP26n3UJyXjLcLNwwyGMw5tT6EEsDv+JBy1sXVCiVSsyZMwetW7dGgwbau6AKxlKXLFkCQxvT3w9dWtfGrC926T1XQh9Y4NC3Vm2M3LdbPe7JuvsY1iux6e4d/jcbKvGuUBGjGjbWa1DxprBcivnzemHvrtlQKJQIeRyPM2cf8eTOsiBfmY1dEZNhZmIFD5tmaO02iwcQbGiECc04oy6bmhfBh0dGV/8LFa2b4Gm26piWNaVpO96WdoMZ39MX3fzqYOq3uzXG3E/cKOzJDH2ajMfRyfh7xSR41/HAzcBolBXT6rRCb8/6eOfcVj7cUeDf6Ifqv0MkiQiWJOBs73d578XVxEiUFSyfomfD2hj3+x6N43fEP0T99+OEFATHJ+PkhxN578W18LJz/HqW7w0/5+b4Nmi5Rs7EjdTCXv2nOTGIzonGikbfoY5dHd5r8dZNKWXjowEBAdi5c+cLyy1cuJCflRQs0dGvd7DSpTmQK5QlehDYevHei+JG9vXB6AF+mPP1Xp43UaDgfs4O1pr7dLB+6T51LS0nB3KlEq7WmvVztbZG0kvGnac088EMH1+MPbAPQcnJGvuUKRQITUnRKB+amoKKdnbQJ4kkm3/pOzlp1o+tpxbrvXgVLJFzzoc70LPPDxg2ci1mzt4KsdgUcXqevZMrl0ApKGAt1jzztRI7lei90CRAKnvKZ4rcT92NsIzzaOY86rmlWcCRI0/nM0bKqtK0HbpqN5j0DFXb4WKv+Tln68mSF7/3xnT3xvhevpj1wz6ExhR+trR5mixBWkY2PN0coU9p+dm87XCxsNXY7mppg+TcF+ffTKrdAtPqtMb4C38iWJL4wrKsxyI1NwuVbUvXu6Mr6dnPjp9tseNna83zIF5kQmtvTGnrg8lb9iMk4cXHj/VYpGZlw8tFv8cvQ54BhaCAvZmDxnZ7M3tIivVeFNe9fA/0qtAbPwR/j5icF58kJuclIUMmhZulO96k1woqZs+ejcOHD+Ps2bPw8HjxmJ2FhQXs7e01ltfB3lRsSqh3g8KuZHYi7tPACwEh2qdNMe/088WEwS3xwdJ9CArXnH7EZnuwWSA+DSurt1lbmaNejQov3OebIFMqEZCYoJFkyfoZWnl64U689imXzDRvX8z2a4FxB/fzKaXF9/kgIQHVnDQbgaqOTniaod8xeblciZCQeDRrVkXj+DVrWhkPHz39z/vPzZUhNTULtrYW8PWpistXHkOflJAjKTcYlWyaFdkqgoe1N+JzCrskX4Yl2pqamD/3dhtxOVia2iNbrhkolhWlbTt01W4UtB1BTxLgW1ez7WDr/mHP/2yx6aeT+7TA7B8P8CmlL+PmZAsHGysk6/mEhLcdaXFo5V5Fs+1wq4q7Kc//oplSuyVm122LiRd28Pu/THkrOzhaWCPpJYGKrskUSjyMTUCLap4ax4+t34t+/vOe1MYHMzo2x9Q/DvD7v4y7vS0crayQVGR2iD4oBAWfElo0yZK1A2w9LDPsufdj00/7VOiHH0N+wJPsl/ccOZk5wUZsqzE7xODDH4Ig4N1338WBAwdw7tw5VK2qmSn8pu08fAufzurJgwOWTDm8lzcsLcxw+FwAv/2zWT2RlJrJr0VRMF108rBW+GLNv4hLlKh7JHJyZcjJU3UT7T5yB+MGtUB0XJpqSumI1jzQuHBTv4mMzG93buOHbj3wIDEe9+PjMbFpM1ibmWHvI1X92G3xmZn47soldUAxt0UrzDl+hCdgsl4NJlsm4wvzy52b+KlnH9x4GoOrMdFoX7kKOlerzodJ9G3Pvhv4eH4fhATHITA4DkMG+cDS0hzHjj3gty9c0AdJyRn47ffz6uTOypVd1X+7utqhenU3nkvBZnowLIBgLWh0dCoqVXTC9KkdERWdgqPH/PVev/upe9CpwkIk5QQjMTcQjZyGQGxiiSDJUX575woLkSVPxrWkX/k665FIzA3mMz9MRWbwsm2BWg7dcCH+R1WdRVbwdR3HcyqyFamwN6uIlm7TIJE9RVSRKadlgaHbju0nbmPJpB4IjExAQEQ8RnVpBisLM/x9WdXFz25LSsvEz/tVny02hXR6/5b45NejiEuWqHs5svNUbQe779R+LXH69mOkSLJ4TsX7Q9ohOjEdVx8WJn7ry6aQa/jOrz/8U+P4tSbG1/KDldgMeyPu89vZbWyq6Pf+quE0NoV0Tv32mHvtAGKy03mvBq+fPB/ZchmsxWZ4t147HI8J4kGEl60TFjTqgieZqbgY//wvujflj8t3sGxwdwTEJsI/hk0pbQorczMcuK06fssHd0eCNBM/nlQlq05u64N3O7fEvN1H8TRdCtdnvRzZ+TK+WJubYWbHFjj58DGSMrPh5ezA8y6iUtNx6bH+j9+JhOOYVHUKIrMiEJEVji7u3Xjy5eVk1XcZuy1Nlob9MXv5OptC2r/SQD4LhOVM2ItVvRx5ylzkKfP4fftVHMBzKlhvh5tFOQzxHI7EvEQ8lKi+T96KoIJ1W+7YsQOHDh2CnZ0d4uNVF4JxcHDgc8/ftNNXg+Fob40pw1rDmV3oKDIJHyzdi7RniZburvYaCTYDuzaGuZkYSz/sr7Gf3/dc4Quz/dANHpgsmNYNttbs4ldPea+GIfIu/n0cDBcrK3zQojUPEFj+w/iD+3iyJlPRTrN+oxs1hoVYjPW9+2nsZ9W1K1h9XTXL4kRYKD49cwozfP2wuENHhKel8Qtf3Yr9770Dr+rsuSCesDl+fFs4O6kufrVg4S6kPbtGhZubPZTKwvq5uNjht40T1esjhjXny737UZj7oWqWhY2NBSZPao9yrnbIyMjFhYvB+H3zBT7Uom+hGWdhaeoIv3IT+MWvkvNCcTh6PnIUqqRRWzN3jQQpsYkV2pX/X3v3ExrFGYdx/NlkN5NunURN6p90A2ItFhGlFAsmh4o9eLGYlgpFeshRpJUWes+heOqhEJTSHppLoTeJFvTQVrBUI3oTY0GkUBPXlhCxm01NNrszZd4kxZFCE33zmnn7/Vz2skz2N7Pz22feed/Jx1qTf0H1eFYPZu/ox/IJs51ErIY6gq3a3n5AQfMaTdcnzfMrrk58rcjBenObnnXv+P7aLa0Lizra12MCQrJU9MPPT+v+wuTNTetDE3wWvbtvl+kdnx17K7WdL8+M6KuzI+Z7+nKpUwd7digsBubZFVdGf9MXw5fNMlPXzo3dVEdQ1Ec73zAPv7r54A8zAjE5O3/V3VVM944jL72mlua8TvUeTm1ncPSiBkd/UiOO9crajXpny26FhVYzifPn3381qz8enXfhyvkbt7Tu+ed0/M29JiD8cm/CjEBMTs8fv81rw1R9772+Sy35vAaPpI/fyQsjOnXhihpRpO2bOtX36g6FrYEmpqq6dPuOBn+4bG4Zu3bt/lUzYbPvxbfNbZDk4VfJCESlPj+ivL6lI9U79m3Yr0JTQce2fZDazpm7wzpbHlYURyoVS+rp7FWxuWhGJ5IwMXz3tFlmupJy8aNn0n+9eWHi3+OGhobU39+/pG0ks7iTRrKn71PlC63y0b3ef99PvtjyXbZ+0J7E4ZPpJZC+eFit65M9l8w8hae5pbBcT9s7FvvG7vdPqLnFz77x5wG3w+6uFa6n53z4qOfQ/MiQb2rVmr7Z/+2S+sayb38AwHLRO4D/B/6hGAAAsIJQAQAArCBUAAAAKwgVAADACkIFAACwglABAACsIFQAAAArCBUAAMAKQgUAALCCUAEAAKwgVAAAACsIFQAAwApCBQAAsIJQAQAArCBUAAAAKwgVAADACkIFAACwglABAACsIFQAAAArCBUAAMCKvByL49i8NuZm5KtoJief1etz8t3Dal0+mlmoa/E8zIp/+kbN477xl7+1JRqzzn9unKtVa/JRbXpuyX0jFzvuLuPj4+ru7nb5JwE8ZmxsTKVSSVlB3wCy0Tech4ooilQulxWGoXK5lb+ir1QqphklO6OtrU2+ob5sc11fcrpPTU2pq6tLTU3ZuftJ37CL+rKtsor7hvPxqOQDPYsrpGTH+/jlWkR92eayvvb2dmUNfWNlUF+2ta3CvpGdSxUAALCqESoAAIAV3oeKIAg0MDBgXn1Efdnme31Z5ftxob5sC1Zxfc4nagIAAD95P1IBAADcIFQAAAArCBUAAMAKQgUAALCCUAEAAKwgVAAAACsIFQAAwApCBQAAkA1/AwY9jnRyohORAAAAAElFTkSuQmCC"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 363
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.814388Z",
     "start_time": "2025-03-24T10:58:40.645080Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# mask\n",
    "mask = torch.Tensor([[0, 0, 1, 1], [0, 0, 0, 1], [0, 0, 0, 0]]).reshape(1, 1, 3, 4)  #手工构造mask\n",
    "outputs_masked = mha(query, key_value, key_value, mask)\n",
    "\n",
    "fig, axis = plt.subplots(*outputs_masked.attn_scores.shape[:2])\n",
    "for i in range(query.shape[0]):\n",
    "    for j in range(outputs_masked.attn_scores.shape[1]):\n",
    "        axis[i, j].matshow(outputs_masked.attn_scores[i, j].detach().numpy())\n",
    "        for x in range(outputs_masked.attn_scores.shape[2]):\n",
    "            for y in range(outputs_masked.attn_scores.shape[3]):\n",
    "                axis[i, j].text(y, x, f\"{outputs_masked.attn_scores[i, j, x, y]:.2f}\", ha=\"center\", va=\"center\",\n",
    "                                color=\"w\")\n",
    "fig.suptitle(\"multi head attention with mask\")\n",
    "plt.show()"
   ],
   "id": "e8bf4c6e13222a0b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhUAAAG6CAYAAAC/RrTYAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAekpJREFUeJzt3Qd4U9X7B/BvmnTvXQotZe/ZspEtQwRlqDiZTlAR/eEf3ANxK4pbGQ4UZSrIlr03FGih0EX33itN/s85pWlTWmjxktD4/TxPoPfm5Oa+Obkn7z33nESl1+v1ICIiIvqXrP7tBoiIiIgEJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFXRLmjRpEoKCgmpV9vXXX4dKpbpuuQEDBqB9+/a4lYgYRaz/FXWp11tBbd9blcumpqbCEurJycnJ3LtB9RCTCqoX8vPzZaO9Y8cOc+/KLe2dd97BmjVrrlq/b98++fplZmbe9H2Ij4+Xz3XixAn8l15jImJSQfUoqXjjjTeqTSpefvllFBQUmGW/6lNSIV4/UyUV4rmqSyq+++47hIeHo76o7r3FpIKoZppr3EdUL2g0GnmjW5+1tTXqE763iOqGPRVUa+XXjM+fP4+HHnoIrq6u8Pb2xiuvvALxY7exsbG466674OLiAj8/P3z00UdGj1+yZIl8fFRUlNF60fsg1td0aUOUF88jiDNgUVbcxP5U3q/aOnv2LAYOHAgHBwc0bNgQ77///lVlioqK8Nprr6F58+awtbVFQEAAZs+eLddXtnjxYgwaNAg+Pj6yXNu2bfHVV19dtT3x+rz99tto1KiRfF7x/GfOnKn1Pn/44Yfo3bs3PD09YW9vj+DgYKxYscKojHgN8vLysHTpUsNrJK6Ni9fnf//7nyzTpEkTw32V6+Hnn3+W2xTb9vDwwIQJE2R9Vjcm5Vqvn6jDbt26yb8nT55seC5R9zWNqRD7/Pzzz8vXWLyGrVq1kvFW/QFlsZ0ZM2bIXgKxH6Jsu3btsHHjxmu+dmI7Xl5emDVrlmGdTqeDm5sb1Gq1Ue/Ne++9J5OI3Nzcat9bNb3GlYntiXVi++IYEa+D6Gm7nvLX99SpU+jfv798fcX7r7yed+7ciR49esg6Eq/R1q1bjR4fHR2Np556St4nyoj3yj333HPV8VZSUiKPoxYtWsDOzk6W69u3L7Zs2XLN/RM9T+I4FPtZ/voQVcWkgursvvvuk43yu+++Kxs58WH56aef4vbbb5cfMqJhFo3hCy+8gF27dv3r5xMNWfkH9ZgxY/DTTz/J29ixY+u8rYyMDAwfPhydOnWSSU/r1q3x4osvYsOGDYYyIrbRo0fLD7ZRo0bh888/x913341PPvlExl6Z2K/GjRtj7ty5cnvig1E07F988YVRuVdffVUmX+J5P/jgAzRt2hRDhw6VH1C1sWDBAnTp0gVvvvmm7H4XH3ziA2P9+vWGMuI1ER+0t912m+E1evzxx+XrdP/998syIoby+8oTtXnz5uGRRx6RHzIff/wxZs6ciW3btqFfv35XXS653uvXpk0buY/CY489Zngusa2aPvDFay32S2xXPL/4UBRJUOUkoNyePXvk6yuSHpHMFBYWYty4cUhLS6vxtRMf/H369DF6L4oP7qysLPn33r17Det3794tX+eaBinW9BpXdu+99yInJwfz58+Xf4uESnyI14Z4fe+88055XIn4xHOJWJcvXy7/v+OOO+RxJ94348ePl89T7vDhw/Iylyj32Wef4YknnpD1KJKAykmNSJTE/ojEcOHChXjppZcQGBiIY8eO1bhfYtsieRavjahrDuKkGumJaum1114Tp476xx57zLBOq9XqGzVqpFepVPp3333XsD4jI0Nvb2+vnzhxomHd4sWL5eMjIyONtrt9+3a5XvxfTjyucePGhuWUlBRZRuxDTft1Pf3795flfvzxR8O6oqIivZ+fn37cuHGGdT/99JPeyspKv3v3bqPHf/311/Lxe/fuNazLz8+/6nmGDRumb9q0qWE5OTlZb2Njox85cqRep9MZ1s+dO1dur/JrVJOqz1NcXKxv3769ftCgQUbrHR0dq93eBx98UO1rHxUVpVer1fp58+YZrT99+rReo9EYra/t63f48GFZTtR3VVXrdc2aNbLs22+/bVRu/Pjx8j0VERFhWCfKidex8rqTJ0/K9Z9//vlVz1U1fhFndna2XP7ss8/kfnTv3l3/4osvynWlpaV6Nzc3/XPPPXfN91ZNr3F52SlTphitHzNmjN7T01N/PeWv77JlywzrwsLC5Drxfjxw4IBh/aZNm656jat7L+7fv/+qOuvUqZN8L16LiE/EKezZs0fv4uIiH1NYWHjdOOi/jT0VVGfTpk0z/C26j0NCQuQZ59SpUw3rRdevOOO8dOkSbiXiDEtcuilnY2OD7t27G+3nH3/8Ic+4xVm4mB5YfhNnasL27dsNZUU3czlx5ivKia5rsb3yM2HRTV1cXIynn37aqCtd9AjUVuXnEWezYtvibPlaZ5e1sWrVKtkzI86oK8cqLl+JnovKsdb29auLv//+W76HnnnmGaP14nKIeE9V7kEShgwZgmbNmhmWO3bsKC+3Xe/5xWtVWloqz+TLeyTEOnETfwuhoaGyZ0as+zdED0HV5xY9KdnZ2dd9rHh9RU9DOXEMiWNJvB9F70W58r8rx135PSIucYjnFD2G4vGV3ydiWVx6u3DhwnX3R9T/sGHDMHjwYPleET0nRNfCpILqTHSVViauG4trs+K6ddX14gPwViLGNFQdf+Hu7m60n6KxFY2uuDxQ+dayZUt5f3JysqGs6DoXH3SOjo6ysRblxKUQoTypENe6BfEhXZkoK567NtatW4eePXvK11mMeSi/JFT+HDdKxCo+vMW+VY333LlzRrHW9vWrC/Ha+Pv7w9nZ2Wi9+BAtv/9a773aPn/Xrl3lGIXyBKI8qRCXZY4cOSIvo5TfJ8YX/BtV97G8jmvzGlX3+orjSFxWq7qu6jbFLBVxma18bIo4HkU9ikSp8vtEXJ4S68T7uUOHDvJSk7gcVJV4TUaOHCkvefz+++8ygSS6Hg5rpjoTZ5a1WSdUHmxX02BKcQZpKrXZT3HmLhpbcX2/OuUN/MWLF+UZnOjREGXFetHwirNvMUZAbEcJ4sNOjDsQH4BffvklGjRoIGdRiEGiy5Yt+1fbFvso6kX0CFT32lS9dl6b1+9mutHnF6+XOLsX4yoiIiKQmJgokwpfX195Vn/w4EH5Oou6LB9rYup9vNZja7NN0RMm3hOiB6xXr14y8RB1K3o+Kr8XxftIvHfXrl2LzZs34/vvv5fv16+//tqoF1IkJmIMhygnBsOKsR5E18Okgkym/Iyt6uC/qmej1anL7I5/S3Svnzx5UiYM13rev/76S84G+fPPP43OTqteMhADOct7BcQAzXIpKSm1OntduXKl7KHYtGmTUfez+ACpqqb9rWm9iFV8MIlZIeU9Mf9WXepKvDbi8pAYcFi5tyIsLMxwv1JEEiEGEYvnE2fxIoEQ+ypmkIiEQtxq88FpyvdiXYhZIhMnTjSadSV6G6r7bhLR2yVmpYibmMkhEg0xgLNyUiHi/OWXX+SMLjEoWCSeYtAn0bXw8geZTPm18Mqj8EUvxbfffnvdx4qua8EUX94kxhfExcXJL2qqSnQxl8/YKD97rHy2KLqZq37Yi8sj4kxZzCKpXFbMmKkN8Tyiga/coyOmCVb3BUziMkx1r5FYL1S9T8wMEdsXswGqnkmL5WvNqqhJTc9VHXEmLOISsxAqE2fOIuYRI0ZAyaRCJIHidReXOMqTg/KZHOJLu2oznqKm19jcRD1WrUPxnqvaE1i1TkVvlBh7UXW6tCB63sRYCjFNWMyEOnTo0E3ae7IU7KkgkxFnhGJcwJw5c5Ceni7Pln777TdotdrrPlYMQhPfASGm1okzavFYMaf/ZvyWx8MPPyyvIYsBd6LXQUxHFA2zOHsW60WPgRicKqaEikZXNLZiWqE44xOJiPjOioSEBMP2RHe6mF4rphiKM2HxQXr8+HF55ld1HEp1xHVtcXlFTLl84IEH5DgHMWVVfBBUvRYuvmtCnImL8mKsguiBEN3+Yr0gpg+K7nCR5Ij9FomemBIs6kQkKmLqrOgxiIyMxOrVq+W0ULHvdSG2KcaXiO50sS3xISz2QexLVWIfxNRGsV/i+cVUVdElL7rcRTd+5UGZ/5a4JCCm4opv9BRxlRNn6eVTlmuTVNT0GpubeG+J5Ehc9hDHyv79++V+iu+hqEzcJ3ocRBziOBJjSkQvh/gOkJqOPTGmRwxUFkme+L6MW+03dOgWYu7pJ1R/lE+ZE9M7a5p+VnWKXLt27YzWXbx4UT9kyBC9ra2t3tfXV06r3LJly3WnlAr79u3TBwcHy2mFlaeX1mVKadX9qem5xJTN9957T5YX++ru7i6f+4033tBnZWUZyv3555/6jh076u3s7PRBQUHyMYsWLbpq+qaYrige26BBAznVdsCAAfrQ0FD5vLWZUvrDDz/oW7RoIfeldevWciphdXGLKYj9+vWTz1F1uupbb72lb9iwoZyeWHX/Vq5cqe/bt6+sR3ETzzF9+nR9eHj4Db1+a9eu1bdt21ZOS6089bG6sjk5OXIap7+/v97a2lrGKaaAVp5+K4jtiH2qqravodCtWze5nYMHDxrWXb58Wa4LCAi4qnxdXuOajo+aplJXVdPrK+Krbgpo1ddDTOOePHmy3svLS+/k5CSnNot9rfr6iOm7YiqtmD4rYhB1LaYOi/f8tY7p1NRUWadiCvGFCxeuGQv9d6nEP+ZObIiIiKj+45gKIiIiUgSTCiIiIlIEkwoiIiJSBJMKIiIiUgSTCiIiIlIEkwoiIiJSBJMKIiIiUgSTCiIiIlIEkwoiIiJSBJMKIiIiUoRFJxXiR5eCgoLkz0aLH/yxpF/YE7/0KX6MSfygkfi1xep+sbK+Ej+8JX4VUfwYlfhxLvEjV+JHoCyF+PGqjh07wsXFRd7ED12JHxejW4elth2W3G4IbDvMz2KTCvFrlrNmzcJrr72GY8eOyV8/HDZsmPyFR0sgfn5bxCQaP0sjfgVx+vTpOHDgALZs2YKSkhL5i6DlPzle3zVq1Ajvvvsujh49Kn8hUvz641133YUzZ86Ye9fIwtsOS243BLYdtwC9hRK/wlf5F/zEr0SKX0GcP3++3tKIaly9erXeUiUnJ8sYd+7cqbdU4ldQv//+e3PvBv2H2g5LbzcEth2mZ5E9FcXFxTKTGzJkiGGdlZWVXN6/f79Z943qLisrS/7v4eEBS1NaWorffvtNnkmJrkwyL7YdloVth+lpYIFSU1PlC+7r62u0XiyHhYWZbb+o7nQ6HWbOnIk+ffqgffv2sBSnT5+WDUFhYSGcnJywevVqtG3b1ty79Z/HtsNysO0wD4tMKshyiOujoaGh2LNnDyxJq1atcOLECXkmtWLFCkycOFFeD76VGgei+oxth3lYZFLh5eUFtVqNpKQko/Vi2c/Pz2z7RXUzY8YMrFu3To5YFwOULImNjQ2aN28u/w4ODsbhw4exYMECfPPNN+betf80th2WgW2H+VjkmArxoosXe9u2bUZdYWL5Vrr2RNUTY8hEoyC69f755x80adIElk68P4uKisy9G/95bDvqN7Yd5meRPRWCmBImuoVCQkLQvXt3fPrpp3JAy+TJk2EJcnNzERERYViOjIyUXWJiQFJgYCDqe7flsmXLsHbtWjnfPDExUa53dXWFvb096rs5c+ZgxIgRsp5ycnJkrDt27MCmTZvMvWtk4W2HJbcbAtuOW4Degn3++ef6wMBAvY2NjZwmduDAAb2l2L59u5wqVfU2ceJEfX1XXVzitnjxYr0lmDJlir5x48byfent7a0fPHiwfvPmzebeLfoPtB2W3G4IbDvMTyX+MXdiQ0RERPWfRY6pICIiItNjUkFERESKYFJBREREimBSQURERIpgUkFERESKYFJBREREimBSQURERIqw+KRCfH3p66+/fkt9jamSGF/9Zunx1VeWXi+Mr34ruoXjs/gvv8rOzpZf0Sp+0c3FxQWWhvHVb5YeX31l6fXC+Oq37Fs4PovvqSAiIiLTYFJBRERE9fNXSsXPtMbHx8tfkFOpVCbpJqr8v6VhfPWbqeMTVzvFrxv6+/vDyqr+nFOw3VAW46vfsm/hdsPkYyouX76MgIAAUz4lEVURGxuLRo0aob5gu0FUP9oNk/dUiDMNYeqGO2HjaA1L9I7fSViyMS07mHsX6AZpUYI9+NtwHNYX5ft78WgAnJ3qTw9LXVir1LBkbDf+G+2GyZOK8q5LkVDYOllmUuHibJmNXjmNyjLr7T/hSr+kKS4hKKl8f0VCYanHl7XKMuMqx3bjv9FuWPa7mIiIiEyGSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBREREf03k4o+XkPxStvP8X6nnzCz5dsIdGhWq8d1ceuNT7osx5QmL9RY5p6AabJMP+87YDYOD0LlvR0q31CoPFYA1h1rLms/FlZ+F4xu4nFXUTeDyu1rqHyOQeVzEirPlYBVA5jD6KeG4adLX2B9/i/4bP87aNWt+TXL9xvfEz+c/VSW//bkR+g+ostVZSa+cR9+i/sW6/J+wXubX0HD5n4wF0uPrz6zcngYNt67YON3Dtaeq6C6xrFlZT8Otg0uGd3E46pSO82Ejc8B2PidhbXHT1Cpg2BJbYfK6VmovPdC5XsaKvclgLoxzMXSj63RFhLfDSUVX3zxBYKCgmBnZ4cePXrg0KFDMIXObr1wd8NHsClxJT4K/z/EF0Tj8WZz4aRxuebj3G28MbrhQ7iYe3WjUK6Dazc0dmiBzOJ0mI3dHVA5z4U+dyH0qXcD2nNQuS8CrDxqfIhelwNdci/DTZ/S37iAOhAqz18B7SXo0x+CPm0U9LlfACiCqfW/tzce/2gifn7zDzwZ/CIunYrG/I0vwc27+vpr26sl5i6biY2L/sGTXWdj79pDeH31bAS1CzCUuW/2Xbj76RFY8OS3eLrnHBTmFWH+xpdhbWsNU7P0+JRgrrbDym4kNC5zoc39DCWpo6DXnoO1x1LAyvOax1ZRUnfDrTj5NqP71Y6PQ+04Cdqsl1GSOhZ6fT6sPZYAsIFFtB2OjwEOj0Cf/Sr0aeMBfQFU7ovNEp+lH1v9LSi+OicVy5cvx6xZs/Daa6/h2LFj6NSpE4YNG4bk5GTcbAN8RmJ/2jYcSt+BpMI4/BH7PYp1xejhObDGx6igwsONn8bGhD+QVpRUbRlXa3eMbTQZP0d/Dp1eC3NROUwB8pcDBSuB0gh5MIsDGfbjr/EoPaBLrXRLM96m03NA0U7oc98HtGeB0hig6B9AZ/rkadxzd2LD99uwackOxJy7jAVPfIui/GIMmzKo2vJjnhmJwxtP4I8P/0RMWByWvrocEccu4a4ZwyvKPDsSv8xbif1/HkHk6Ri8N3EhPP3d0efubjA1S4/v3zJn26F2nApd/nLoClZAr42QiYA4ttT299Th2Eqtss3JKM1dCF3RVui1YdBmvgCofWFlNxQW0XY4TIQ+90ugaBugDYc+63+A2gewux2mZunH1jgLiq/OScXHH3+MRx99FJMnT0bbtm3x9ddfw8HBAYsWLcLNpFap0cihKc7nnDas00OPCzmnZQ9DTYb5jUeONgsH07fXmHQ82HgGtif/hcTCyzAfa8C6HfTF+yqt0wPF+6Cyvrpby0DlAJX3Dqi8d0Hl9hWgqdxlpgJsB0CvjZJnLSrvA2XdorZDYGoaaw1aBjfFsa2nDOv0er1cbtuzZY3Z+LFtFeWFI5tPos2V8n5NfODZwB3Ht1a8J/Kz8xF2MAJte7WCKVl6fEowV9shji2VdXvoivZWWqeXyyqbax9bNt67YeOzBxr3b6DSVGpn1AFQqX2Mt6nPgb74xLW3WV/ajivxiW1UbDIXKDl57W3eBJZ+bGksLL46JRXFxcU4evQohgyp+FCysrKSy/v378fN5Kh2kYlFTkmW0XqRMLhYu1X7mCaOrWQvxu8x39a43UG+d0GnL8WulA0wKyt3qFSaq86GUJoGWHlX/xhxSSNrDvQZT0IvzpJgBZXH74DVletmVp5QWTlB5fgY9EW7oM+YDH3RZqjcvgCsu8OUXL2codaokZFkXH8ZyVlw96u+/sT6zKrlkzLhcaV8+f9iXdUy7r7Vb/NmsfT4/i1zth3lx5a+yrElllU1HFt67SVos15EScZjKMmcJY8ta88VhmOr/HF12Wb9aju8yv6/apupFfeZiKUfW64WFp+mLoVTU1NRWloKX19fo/ViOSwsrNrHFBUVyVu57OxsmIKtlZ3sgVge+y3ySnOqLdPIvgn6eY/AR2H/h3qp5ETZ7Qp95jGovDZC5TAB+txPK3JG0X2Zv+RKY3IOsO4KlcP90GeZ5no2UV3bDnO1G+X0JcflrZy2+BhsvDdD7XA/SnM/Qb133baDyARJxY2YP38+3njjjX+9nbzSbJTqS+Fs7Wq03lnjiuwS42xM8LT1haetD6Y1nW10qUP4sPMyzD/7HJo6tZGDPF9tLwYulhG9IXc1fBj9vUfgrbNPw2R0GdCL8RxVzwLUnoAupZYb0ZaNmygfoS23WSKvIRsXuwjYBMOUslJzUKothbuvcf25+7giI/Hq+hPEereq5X3dkH6lfPn/ldeVL188GQVTsvT4TE2pdqPysaWy8hIXBQzkch2OLV3JWag0ZbM7yh9XdRtiWSeOQVO6KW3HlR4Ksc3K21B7ASU1D3i/GSz92MqysPjqdPnDy8sLarUaSUnGAx7Fsp9f9VNV5syZg6ysLMMtNjb2hnZUJBSX8y+hpXMHoyShhXN7ROdfuKp8cmE83jv3Aj4Me9FwO5N1FBG5Z+TfmSWpOJK+Cx+EzTYqI2Z/bE/+E19ffAemVQKUnIHKpleldSrAprfRGdO1WQGaloAuudI2T0OlaWJcTDSMpfEwJW2JFuePXkKXwZXqT6WSy2cPnK/2MWf3n0eXQRXlha5DOuLclfKJkclIS8hAl8HtDfc7ONujdY/mOLs/HKZk6fH9W3VtO5RqN8qUQF8SCivb3pXWqeSyvrj2x5bKuhX0pVeOrdJY+bfRNlVOUNl0rsM2b+G240p8qLxNlRNg3akO21SGpR9bWguLr05JhY2NDYKDg7Ft2zbDOp1OJ5d79ar8hq5ga2sLFxcXo9uN2pG8Hj09B6GbRz/42DbE+IBpsLGyxcG0HfL+BxpPx8gG98u/tfoSJBbGGt0KSvNQVFoo/xZJSn5p7lVlxOyP7JIspBQlwNT0+YsAh/sAuzFl3y3h8iagsi8b0S3eaK7vQ+X0fMUDHGcANn3loCpo2kLl+hGgbgh9/h8V28z7Xk43g/29cnopHB4CbAdBn/+LyeNb+ck63DFtMG5/pD8CWzfEM189CjtHW2xaXDaIdvaSGZjyzgOG8qs/W49uwztj/Kw7EdDKHw+/dg9ahjTD2oUbK8osWI8HXhqHXqNCENQ+ELOXzkBafAb2rjnM+G4hdW07lGw3hNK8H2DlMAFW9mOh0jSDxuUtOVCxtGCFvF/j+iHUzv8zlFc7PQ3VlWNLpWkHjdsnUKkbQlewvNI2F0PtNANWtoOh0rSCxu1DoDQJusLNsIi2I38pVE5PyfZCJBxiGxCJRuEWk8dn6cfWSguKr86XP8SUsIkTJyIkJATdu3fHp59+iry8PDmi+2Y7kblfXq4Y3uBeuGjcEFcQhW8uzkeutmzAiru1J/R6Heqtwr+ht/KAyvnZsgFWJeegz5haMdVL7V82qvsKlZUL4Pp2WVldljxb0afdJ6eUGRRtgT77NagcHwdcXgG0kdBnzgBKjpo8vJ2/75PzrsUXsoiBRhdPRGHuiHnITC6rP59A0ZWsN8rG5z+4AJPeuh+T5z2AuAsJeH3M+4g6U3HWuvz9tbBztMPMbx6Hk5sDQveEYc6IeSgpKmF8txhzth26wvXQZntAI6ZYq72gLzmHkvRJhm5+lTy2KrUdKldYu84v6+7XZUNXEoqS1PFGlxJL876RH9wa13cAKxfoi4+gJF3EUgyLaDvyvpXxqVxEOReg+Aj0GVPMEp+lH1s7LSg+lV7MXamjhQsX4oMPPkBiYiI6d+6Mzz77TH6RTW2IAVeurq54ctcY2DrVzy/wuZ6PGhyDJRvm39ncu0A3SPTg7cBaeUnh357934gbbTvK243k8MZwca53XwRcK9YqNSwZ243/RrtxQwM1Z8yYIW9ERHXBtoPIsllmyk9EREQmx6SCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUoYGZ/HWsC6zs7WCJbh9yBpYseUZvWDKfhfvMvQtUg8Enx0PtYAtLpNNZ9jme25YCWDKb26PNvQu3BMt+FxMREZHJMKkgIiIiRTCpICIiIkUwqSAiIiJFMKkgIiIiRTCpICIiIkUwqSAiIiJFMKkgIiIiRTCpICIiIkUwqSAiIiJFMKkgIiIiRTCpICIiIkUwqSAiIiJFMKkgIiIiRTCpICIiIkUwqSAiIiJFMKkgIiIiRTCpICIiIkUwqSAiIiJFMKkgIiIiRTCpICIiIkVoUM880rYLHuvUDd72jjiXnozX9m7DyZTEassOD2qB6V16orGLG6ytrBCZlYnvTh/G6gtnjcrNCu6D+9t0hIuNLY4kxuOlPZsRlZ0Jc2jsMgFNXCfDVu2FnOJwnEl7B1lFodWWbeh0Fzr5zDNaV6orwqaoYKN1LdynI8B5PKytnJFReByhqW8hXxsDc7jvtk6YOCgYXi6OOB+XgndXbEdoTFK1Zcf2ao9R3duieQNPuXw2Nhmf/7XnqvJP3dELY3t1gLO9LU5ExmPe79sQk2Ke+hv91DDc88JoePi54eLJaHzxzCKEH46osXy/8T0x8c0J8AvyRtyFRHz/fz/j0IbjRmUmvnEfRkwbDCc3R5zZG4bPnvoOcRHVv+epZvc27oGJTW+Dp60Tzmcn4r0z63Am63K1ZQf5tcXUZgMQ4OgBjUqNmLw0/BS5B+vjThjKPN5iEIb5d4SfnStK9KU4lxWHheFbEJpZ/TZvtvuCumNis77wuhLfu6HrEZoZV23ZwSK+Fv1kfNYqNaJFfJf2Yt3lk/J+jcoKM1oPQV+flmjk4I4cbSEOplzCgnObkVKUA3O4u1FvTAgcAA8bZ1zMTcCC86sRlh1bbdnbvNvjoaDBaGjvBY2VGpfzU/B7zE5sTjxmKPN/be7DCP9uRo87mBaG2Se+hzmMtpC2o849Fbt27cKoUaPg7+8PlUqFNWvWwFTubNoKL/cagAVH9+HOVT/iXFoKfrrjHnjaOVRbPrOoEAuPH8DYtb9g2Iql+OP8aXzYfwT6NQoylHmiU3dMat8Vc3dvwV1rfkG+tlhu01athqk1cByO1p6zEZHxFfbG3YPs4nB09/sGNlYeNT6mRJeDrdH9DbftsUON7m/qOgVBLg8iNPVN7It/AKX6AnRv8A2sVDYwtWFdWuKFMf3wzcYDmPDBLwiPS8VXT42Fh5N9teVDWjTChqNhmPb5Cjz88W9IysiR5X1cHQ1lJg8Jwf39OuPt37fioY9/RUFxCb56cixsNKavv/739sbjH03Ez2/+gSeDX8SlU9GYv/EluHm7VFu+ba+WmLtsJjYu+gdPdp2NvWsP4fXVsxHULsBQ5r7Zd+Hup0dgwZPf4umec1CYV4T5G1+Gta016hNzthvC0AYd8HybO/DNhX/wwJ4vcD4nEV/2mAR3m4r3UmVZxQX4PmIHJu77Bvfu/hxrLx/F6x3HopdXc0OZ6LxUvBf6F+7Z9Rkm7/sW8fmZ+LL7ZLjbVN8e3UzD/NvjhbYj8M357Ziw6yuEZyfiqx4T4VFTfCX5+P7CTjyy5zuM37kQa2OP4Y1OY9Dbuyw+O7U1Wrs2wLfnd+C+XV9h1uFfEeTkiQXdH4Q5DPTphOktRmNp5BY8evhTXMyNx4edH4WbtVO15XNKCvBz1DZMP/I5phz8CBsSDuPFNvehm0dLo3IHU8MwZvcbhtubob/AHPpbUNtR56QiLy8PnTp1whdffAFTm9YxBL+FncIf50NxITMNc3dvRoG2BPe2al9t+QMJsdgUdQERmemIycnE4tBjCEtPQTe/hoYyUzsEy8RjS3SEvG/W9r/h4+CEoUEtTBhZmSaujyA2ewUu565BbsklmQiU6gvRyHlMzQ/S61FcmmZ0qyzI9WFEZH6L5PztyCk+j5PJc2Gr9oGvw2CY2sMDu2LVvlCsPXgWlxLTZSJQWKzF3T2rr7+5P27E73tOITwuBVHJGXj91y2wslKhe8tAQ5kH+3fFd5sPYcfpS7gQn4qXf9oIb1dHDOrYDKY27rk7seH7bdi0ZAdizl3Ggie+RVF+MYZNGVRt+THPjMThjSfwx4d/IiYsDktfXY6IY5dw14zhFWWeHYlf5q3E/j+PIPJ0DN6buBCe/u7oc7fxGdatzpzthvBQkz5YFXsEf14+hku5KZh3ei0KS0twd4Bxr165o+mR2J50FpG5Kbicn45fo/bjQk4SunhUnJBsjD+Fg2kXEVeQgUu5yfjo3N9wtrZDC2c/mNrDTXtjVcwRrI09LuN7+9RfZfEFdq22/JG0KPyTeO5KfBlYFnngSnyN5f252iI8cWApNieEyuTpdOZlzD+9Hu3cGsLP3tXE0QH3BvbHuriDMjmIzkvCR2ErZXx3VOlpKHci8yJ2p4QiOj8Z8QVpWBm7B5dyE9DBrYlRuWK9FunFOYZbrrYA5jDOgtqOOicVI0aMwNtvv40xY67xQXcTiMsXHbz8sOdytGGdHsCeuGh09fWv1Tb6+Aeiqas7DiaUdU8GOLvKBEJso1xOSTFOJCegq0/ttqkUFTRwsW2LtIIDldbqkVpwAO52nWp8nNrKAQMDNmNg4FYE+34GJ+uKD1N7TSPYabyRWrDfsE6rz0Vm0Sm4XWObN4NGbYU2Ab44EF5x2UWvh1zu2KRBrbZhZ6ORXZnZ+YVyuaGnq0wgDlbaZm5hMU5HJ6JjkGnrT2OtQcvgpji29ZRhnV6vl8ttexqfHVU+2zi2raK8cGTzSbS5Ut6viQ88G7jj+NbThvvzs/MRdjACbXu1Qn1irnZDEJcv2rj642BqRVeyHnq53NGtIkG9lu6eTRHk6CWTjZqeY2xgN3mGLC49mCO+A6mXjOI7kHoRHd0rzlyvpbvXlfjSomos42RtC51eh5ySsuPPlPG1dG6Io+nnjeI7mnEB7VzLkqDr6ereHAGOPjiVWfEaCZ3dmmHNba/jp56zMavVWLhoTN/LpLGwtqPejKlwt7OHxsoKqQX5RuvFcjO3mi8POFvb4OBDT8JGrUapTo9X9m4xJBE+DmVdg6n5eVW2mQfvK/eZio3aHVYqDYqq9DSIZSdr4+y6XF5JFE6nvCovk4jxEk1cJ6FXw5+xO/ZuFJYmyXEZQtXeC7Fcfp+puDvay8QiLce4/sRyE1/3Wm1j5ujbkJKda0hMvFzKGoDqtll+n6m4ejlDrVEjIynLaH1GchYCWlf0jFXm7ueGzKrlkzLlNVWh/H+xrmoZd9+y++j6xOUIkYymF+UarU8rykWQo3eNj3PS2GLT4BdhbaWRH6bzQ//CwdSLRmVu82mFd7vcJy8XpBbl4omDi5FZYvx+NFV8Ip7KxHITJ69rxrfl9v8Z4nvn9DqZiFTHxkqDmW2GYkPcaeRpi2BKrtaOMr6MYuP4MopzEOjgU+PjHNV2WNH3FbnvpXodPg1fhSPpFwz3H0oPx66U00gsSIe/vScebX4H3u88DU8d+Rw6ecpqGq4W1nbc9KSiqKhI3splZ2fDlHJLijFi5VI4WtvInoqXew5ETHaWvDRS32UWnZS3chmFJ9Av4E8EuNyDCxkLYUmmDOmG4V1bYernf6BYW2ru3SELbzeEPG0xJuxeCHuNLXp4NsXzbUfISyGVeysOp12SZdxsHDE2MATvd52Ah/d+jYxi4xOVW5GI796dX8JBY4MeXk3xfLvhMj5xaaQyMWjzg+D75FiYeaf/Qn2RX1qEaYc+hr3aFl09WuCpFqMRX5AuL40I/yRVDLq9lJcoB3/+1mcuOrs3w7GMmgdIkpmnlM6fPx+urq6GW0BA7brjqsooLIBWp4OXvfEZqFhOqdLTUJnIN6OzM3E2LRnfnT6CDZHn8VTnHvK+5CuP86rSK+Fl73jNbd4MxaUZ0Om1sFWXzXQoJ5aLSlNrtQ09tMguOgdH67Iu3fLH2VTZpk0dtqmUjLwCaEt18HQ2rj+xnFqlp6GqRwYFywGZT3y5So6bKJeaXfa4ard55T5TyUrNQam2FO6+xteb3X1ckZFY/UwUsd6tanlfN6RfKV/+f9UzC7Fc9QzE0ijVbggZxfnQ6krhYWs8qE/MAql6dl+Z6GKPzU/H+ewE/BS5F1sTzmBK8/5GZcR1fVHmdGYs3ji1Wp4Rj6lhnMbNUh6fiKcysSx6T64XnxjU+eOlfdgafwZTm/erNqFoYO+Gx/cvMXkvhZBVkifjc7cxjs/dxhnpxdnXjC+uIA0RufFy5sfO5FN4MKj6MQpCQmE6Motz0dDBtL24WRbWdtz0pGLOnDnIysoy3GJjb6yHoESnw+nURPRpWHENTSXHSTTGsaT4Wm/HSqWSl0KE2JwsJOfnyh6Mck7WNujs0wDHkmu/TSWUJQRn4WlflvCUUcnljMKK3ohrs4KzTQsUaVPkUoH2Mgq1KfCy72kooVE5ws22IzJrvU1liITiXGwSerSs+HBQqYAerQJwKjKhxsdNGhyCx4b1wFNfr8bZWOOppHFpWUjJyjPapqOdDTo09sOpKNPWn7ZEi/NHL6HL4A6GdeLMTiyfPVBxLbiys/vPo8ugivJC1yEdce5K+cTIZKQlZKDL4IqBrA7O9mjdoznO7g+HJVOq3RC0crpnPHp4VYw3UkGF7p7NcCqz9lOrRX3aWF17VpHYrricYEoV8TU12g+xfCojtk5tY+V9L08oAh098fiBxcgqMc8gRhHf+Zw4BHu0MIpPjJM4k1UxHq6u8VXlbesKF2sHpBWZtldMa2Ftx01/99va2sqbEr4/dQQfDbgDp1IScTIlAVM6hMDB2lrOBhE+HnAHEvNy8P7h3XJZ9EiIsqKnQkwRHRjQFGNatMXLu7cYtvnD6aN4umsvRGZnIDY7C8936ysTjc1RFdfeTCUy60d09J6HrKIzyCwKRRPXh6BR2cvZIEJH73dQpE1GeMancrm52xNy0GVeSYwcU9HUbTLsNf6IzVlp2GZU1k9o7vYY8kqiUVAShxYeM1BUmoyk/G0mj++n7cfw1kPDcCY2GaHRiXhoQBfY21hjzcEz8v63HxqG5KxcfPbXXrkseifEd1D839INiE/LNvRI5BeVyKmjwi87j+HRYT0QnZIpk4zpI3vLROOfU9VfG76ZVn6yDrOXTMf5IxcRfigCY2aOhJ2jLTYt3i7vn71kBlLj07Fo7jK5vPqz9fhoxxsYP+tOHFx/DAMm9EHLkGb49PFvDNtcvWA9HnhpnJyHnhCZjElv3oe0+AzsXXMYlkzJdkP4OXIv3uw0Dmcz4xCadRkPBPWGvcYGa2OPyvvf6jQeyYXZ+Dx8s1ye0qwfzmTF4XJeOmzUGvT1bomRDTtjfuif8n4xhmJa8wHYmRSG1KIcuFk74N6gnvCxc8GWhOq/V+Zm+unSPrzVeSzOiPgy4/BQ016wV9tgTUzZ9zK83XmcjO+zsLK2b0rzfvK1ED0VYszBbT4tMLJRZ8PlDZFQfBgyQQ4AffrQz7BSWRl6QsR0W/FBb0qip2FO2wkIy76MsOwYjA+8TcYnZoMIc9tOQEpRFr67uEEuP9h4EMJzYhGXnybj6+HVBkP9gvFxeFnbKB47sclQ7Eo+JWd9iDEVTzS/U/ZsHE4zfcK+0oLajjonFbm5uYiIqLjeFBkZiRMnTsDDwwOBgbUbSX2j1l0Kh6e9A2aF9JEDKcUljUf+XmEYvOnv5AydmFJwhYPGGm/3vR0NHJ1QqNXiYmY6Zv6zXm6n3NcnD8ly828bduXLr+LwyIYVKCo1/XX7hLyNcsBmS/cZsNF4IacoDIcSnzAMtLTXiFkSOkN5a7ULOni9LstqS7ORVXwW++MfktNRy13KWgS1lb0sp5FffnUMhxOfgE5fbPL4Nh0/D3cne5koiIGU4ZdT8NRXq5F+5fKHn7tx/d3TpyNsNBp8PHWU0Xa+2rAfX28omyWzeOsRmZi8OmGI/PKr45fi8dRXq8wy7mLn7/vkvHLxhTNiINXFE1GYO2IeMpPLBlT5BHpBr9MbnW3Mf3ABJr11PybPewBxFxLw+pj3EXWm4uxy+ftrYedoh5nfPA4nNweE7gnDnBHzUFJUllTVF+ZsN4TNCafld1I82XIwPG2dEZ6dgOmHliD9ytgHMU2y8nvPTm2Due1Hw8fOFUWlJYjKS8HLJ/6Q2xFE2SAnb4xq1FUmFOJ7H8QH+pT938nppaa2KT5UxvdUq8Hyy69EfE8d/NE4vkpth73aGnM7jIKvvYuMLzI3FS8dXyG3I4jkaKBfG/n3H/2nGz3X1H0/XDXu4mbbnnwSbjZOmNJ0GDxsnRGRE4//nfjeMHjTx879qvp7rtVYeNu6oUhXgpi8ZLx9ZpncjiAuUzVzaoDhDULgpLFDalE2jqSfxw+XNsovMjO1nRbUdqj0Yu5KHezYsQMDBw68av3EiROxZMmS6z5eDLgS10gbffomrOztYIm+GrIUlmzOp1NhyXwW7oOl0upLsANr5SUFF5fqv1jnZlCq3ejw+/NQOyjXg3Er0eks+1cT3BzMc/nEVGxur/2lGEtuN+rcUzFgwAA5h5aIqLbYbhD9N1h2akxEREQmw6SCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUoYGZ2KSpYWWnhiX6M6MLLJlthg6WLPeeHrBU2pJCYPVa1FdZF91hZWdn7t2gG5ANN1i0jxvAUukKC4E5tWs32FNBREREimBSQURERIpgUkFERESKYFJBREREimBSQURERIpgUkFERESKYFJBREREimBSQURERIpgUkFERESKYFJBREREimBSQURERIpgUkFERESKYFJBREREimBSQURERIpgUkFERESKYFJBREREimBSQURERIpgUkFERESKYFJBREREimBSQURERIpgUkFERESK0KCeeTC4E6b1CIG3kyPCklLw5ubtOJWQWG3Zezt3wN0d2qCll5dcDk1Mwsc79hqVf+/OYRjbsZ3R43ZdjMLU5atgDv29h2Co7x1wsXbF5YJYLI/5EVH5l677uBD3npjWdDpOZB7F1xc/Nay/s8EYhHj0hLu1J7R6LWLyI7E2bgWi8i/CHMYP6YyH7giBp6sjLsSm4MMf/8HZS9XXX9OGnnhsXG+0DvKFv7crPv55O37bdOyqct7uTphx323o3bEJbG01uJyUibe+24RzkUkwtXFDO+PBUd3g4eaIiOgUfLx4G85erD6+Jo088ei9fdC6iS8a+Lji06X/YPnfxvFNHd8b0+7pbbQuOi4NE2YtvqlxWKKHO3bGYyEh8HZwxLnUFLy+/R+cTKq+bia074CxbdqipWdZ23E6OQkf7t1jVP6DocMwvm17o8ftjIrEpDXmaTsYXwXGB7PFV6ekYv78+Vi1ahXCwsJgb2+P3r1747333kOrVq1gCne0aYm5g/vj1Y3bcDI+ARO7dcWiCWMx9JvFSM8vuKp8j8BGWHcmHMcvb0dRqRaP9eyGxfePxR3f/oik3FxDuZ0XI/F/6zYZlotLS2EOwe49ML7RA1gWsxhReRcxyGc4nm4xG6+fmY0cbXaNj/O08cK4RvfjQk7YVfclFSbit5gfkVqUDGsrGwz2HY5nW87GK6EvIFebA1Ma0qMVZj7QH+8u3oozFxMwYXgwPps9DvfMXoSM7Kvrz9ZGg7jkLGw7dB7PPTig2m06O9jiu1cm4Oi5WDz74Spk5uQjwNcd2XmFMLXBvVrhmUcG4P3vt+LMhQTcd0dXfDJ3PCY8J+LLv6q8na014pOy8M+BcDz7yMAat3sxNhXPvPW7YblUp0d9Y+62Y2TLVnipX3+8/M9WnEhMwJQuwVg6ZhwGL12EtIJq2o5GAfgzPAxHE+JRpC3FEyHd8OPYcRj641Ik5VW0HTuiIvG/zRvN3nYwPmOMz3zx1enyx86dOzF9+nQcOHAAW7ZsQUlJCYYOHYq8vDyYwpTuwVh+IhQrT51BRGo6Xt2wFQVaLcZ3Ms7Gyj3/5wYsO3YS55JTcCktA3P/3gIrlQq9ggKMyhVrS5Gal2+4ZRcWwRyG+I7A3tQd2J+2GwmF8TK5KNEVobdnvxofo4IKU5o8ib/iVyG1KOWq+w9n7EdYzhmkFqcgoTAOK2J/gb3aAQ3tjV8DU3hgRDDW7DiNdbvPIDI+He8u3oLCohKM6teh2vKip+Hz33Zhy4FwFJdUfzA8cmd3JKfnyJ4J0eMRn5KNg6HRMhkxtftHhuDPbaexfkcoouLS8P73W1BUXII7B1b//jx3MRELf9mJrfvCUVJDfEJpqQ7pWfmGW1bO1Y3Mrc7cbce0rsFYHnoaK86eQUR6Ol7atgUF2hLc0676995zG//Gz6dO4lxKCi5lpOP/tm6Wx1qfwECjcqKRTs3PN9yyi8zTdjA+Y4zPfPHVqadi48aKjEdYsmQJfHx8cPToUfTrV/MHnxKsrazQroEvvt5/yLBOnK/ti4xGl4YNarUNe2sNNFZqZBUan8X2aNwIB559Qq4/EB2LT3buRWaBac901So1Ah2CsDHhL8M6PfQ4l3MGTZ2aAzX05I9sMAY5JdnYl7YTLZxaXfc5bvMehHxtHi7nx8CUNGoreRlj6V+V6k8PHD4Tgw7Na1d/1bmtazMcPB2F+U/fiS6tA5CSnosV205g7Y7TMHV8rZr64sc1B43jOx2D9i38/9W2A/zc8edXT6C4RIvQC/H4atluJKWZtpfp3zJ329HexxdfHjZuO/bGxKBrg1q2HRoNrNVWyKzSdvRs1AiHH3sS2YWF2Hc5Bh/t23tVmZuN8V0f4ys0WXz/akxFVlbZ2aCHhwduNncHe2isrGRPQmVpeflo5lm75//fwNuQnJuLvZEVH6i7LkVhU/gFXM7MRqC7K54f0Bff3zcW9y79FTrxqWAiThpn+aGfrTU+wxYJg59d9R9KzRxboo9Xf7x99qVrbruDa2dMbTIdNlY2yC7JxIIL7yGvtKKLzBTcnO3lB296lvGZaXp2Phr73/j7p6G3K8YO6oRlG49i8Z+H0LapL55/eCC02lKs33MWpuLmUkN8WXn/Kr4zEQl4+6sNiI5Ph5e7E6aO64Wv3rgfD72wGPmFJaivTNp22F9pO/KN60acuTWr5fO/2LcfknLzsCcm2rBuZ1QUNkVEIDYrC4Fubvhf775YcvdYjF1u2raD8V0f43MzWXw3nFTodDrMnDkTffr0Qfv21XfvCkVFRfJWLju75rEBN9NjvbphZNvWeOjn342uK60/G274+3xKKsKTU/HPU1Nl78X+qFjcqmyt7DC5yRP4OfqH6yYI4TnnMO/cSzJx6es1EI82fRrvhb1+zXEa9YWVlUpeJvnqjz1y+Xx0Mpo18pKJhimTipvlwIlIw98XY1LlWI3VXzwmx2/8tT0U9VFt2o5bpd0QngjpjlGtWuH+FcZtx7rzFW1HeFoqwlJSsGvKNPRsFIB9sabtCfw3GB/juyWmlIrro6Ghofjtt9+uO0DL1dXVcAsIuLFr+Rn5BdDqdPBydDBa7+nogJTrXJed2iMYj/fqhsm/rkR4Suo1y8ZmZiE9Px+N3d1gSmLQZKm+FC4aV6P1ztYusnehKm9bH3jZeuOp5rPwRdcl8tbDsw86unaRf3vZ+BjKFuuKkFKUjMi8i/gp+nvo9KXo7dUfppSZUwBtqQ4ero5G6z1cHJCWeePX1VMz8xAZl2a0Lio+Hb6ezjClzOwa4nN1/FfxVZWbX4SYhAw08nNHfVWbtkOpdkPIKLjSdjgY142Xw/Xbjke7huDJbt3wyKqVCEu9TtuRnYU00Xa4mbbtYHw1Y3ymj++GkooZM2Zg3bp12L59Oxo1anTNsnPmzJFdneW32NgbO/sv0elwJiEJvYIqBqKoAPQOCsTxuIQaH/dozxBM79MTU39bLaeUXo+fsxPc7O2RnGuaAWTlREIRkx+F1i5tDevEwJvWzu1wKTfiqvKJhQl488wczDv7suF2Kus4zoteibMvI6PE+IO2MpVKBWuVaWcTiw/csKgkdGtbqf5UQEi7QJyOqLn+rufU+Tg0bmD8ARvo545EE485EPGFX0pCSIcq8bUPlOMglGJva41Gvq5IzTDt5Sul1LbtUKrdKG87QpOT0CegStsREIhjCTW/9x4P7oYZPXpi4upVcsre9fg5Ocmu7Ot9ECiN8VWP8Zknvjp9suj1ejz99NNYvXo1duzYgSZNmlz3Mba2tvKmhEWHjuL9UcMRmpCEU/GJmNS9K+ytreVsEEHcl5STi492lHWFiymkz/brhVlrN+ByVpahlyO/uAT5JSVwsLbG07f1wqawC/KFFmMqZg/sh+j0TOy5VHFtylS2Jm3ApKDHEJ0XKb+bYpDPMNhY2WJf2i55/6Sgx5FZnIE18b9Dqy9BfOFlo8cXaPNljZavF48d4Tcap7KOIaskU17+EN+D4WbtjqMZFYOCTGXZhqN47bHhOBeZiDOXEjFhWFf5IbluV1k3/uuPD0dyRi6+/L2s/sQYhSYNPeXf1hq1/D6KFoHeKCgsweXkst4bMZbih1fvx6RR3bH14Hm0a+aHuwd2xDuLNps8vl/XH8ErT41A2MWksimzdwTLaaPrdpTF9+r0EXIg6Ve/7q6Ir1FZfBoZnzNaNL4SX1JZfE8/1B97jl5EQmq2jF98Z4WYUrpl79XTh29ldW07lGw3hO+PHcVHQ4fjVFIiTiYmYkrXrvL4X3G2rG7EfYl5ufhgb9l77/GQbniuZ2/M3Pg3LmdnybNGQbQb5W3Hsz16YUPEBaTk56Gxqxv+r28/RGdmYFd0lGL7zfgYX32LT1PXbstly5Zh7dq1cHZ2RmJi2RdtiO5JMff8Zvv73Hl4ODjg2X694e3ogHNJKfJLqsRgTcHfxVk2XuXu79oRNhoNFo4bZbSdz3bvx+e796NUr0crHy+M6dAWzna2SM7JxZ7IaHy6a59Z5isfzTgIZ40zRvmPu/LlVzH4/MIHhrEPHjaeRvFdj06vg59dA/TyfAaOGmfkaXMRnX8JH4a/LaeXmtrWg+Fwd7bHY+P6wNPVAedjUvDsByvlYE3B19PFaACR+BD9Zd4jhuWHR3aTN/GdFE++U/a9DWI8xewFf+Kpe/ti6t29EJ+SJb8ka9M+03/obtsfDncXB0y7tw883RxwISoFz81fgYysSvFV+o4JLw8n/Pj+RMPyg6O7yduxM7GY/uZyuc7b0xlvPHMnXJ3t5CWWk+FxePTlX+TlpPrE3G3H+vPh8LS3x6xefWQDLL5caNKalXIwnODv4gKdHHNf5qGOnWCr0eCrO0cbbefTA/uw4MB+mdi19vbG2Lbt4GJri+S8XOyOjsbH+/eape1gfIzvVolPpa/Dp5ToNq/O4sWLMWnSpFptQwy4Eg1J01fmwcrODpZo8LDjsGTHPu0MS6Yurn9fLlVb2pJCHFn9iryk4OLiYrLn/bdtR3m7ETj/bYttN4huVbrCQsTMeblW7UadL38QEdUV2w6i/wb+oBgREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESlCAxPT6/Xyf11RISxVcW4xLFlpseXWnaAvKXuPWqLSkkKj47C+MLQbhZb93iO6FZUfd7VpN1R6E7culy9fRkBAgCmfkoiqiI2NRaNGjVBfsN0gqh/thsmTCp1Oh/j4eDg7O0OlUt3058vOzpaNkXgxXFxcYGkYX/1m6vjE4Z6TkwN/f39YWdWfq59sN5TF+Oq37Fu43TD55Q+xQ+Y4QxIvvCW+ucoxvvrNlPG5urqivmG7cXMwvvrN5RZsN+rPqQoRERHd0phUEBERkSIsPqmwtbXFa6+9Jv+3RIyvfrP0+OorS68Xxle/2d7C8Zl8oCYRERFZJovvqSAiIiLTYFJBREREimBSQURERIpgUkFERESKsOik4osvvkBQUBDs7OzQo0cPHDp0CJZi165dGDVqlPyGM/ENg2vWrIGlmD9/Prp16ya/PdHHxwd33303wsPDYSm++uordOzY0fDFNb169cKGDRvMvVv0H2g7LLndENh2mJ/FJhXLly/HrFmz5LSbY8eOoVOnThg2bBiSk5NhCfLy8mRMovGzNDt37sT06dNx4MABbNmyBSUlJRg6dKiM2RKIb4Z89913cfToURw5cgSDBg3CXXfdhTNnzph718jC2w5LbjcEth23AL2F6t69u3769OmG5dLSUr2/v79+/vz5eksjqnH16tV6S5WcnCxj3Llzp95Subu767///ntz7wb9h9oOS283BLYdpmeRPRXFxcUykxsyZIjRbweI5f3795t136jusrKy5P8eHh6wNKWlpfjtt9/kmZToyiTzYtthWdh2mJ7Jf1DMFFJTU+UL7uvra7ReLIeFhZltv+jGfp1y5syZ6NOnD9q3bw9Lcfr0adkQFBYWwsnJCatXr0bbtm3NvVv/eWw7LAfbDvOwyKSCLIe4PhoaGoo9e/bAkrRq1QonTpyQZ1IrVqzAxIkT5fXgW6lxIKrP2HaYh0UmFV5eXlCr1UhKSjJaL5b9/PzMtl9UNzNmzMC6devkiHVz/Oz1zWRjY4PmzZvLv4ODg3H48GEsWLAA33zzjbl37T+NbYdlYNthPhY5pkK86OLF3rZtm1FXmFi+la49UfXEGDLRKIhuvX/++QdNmjSBpRPvz6KiInPvxn8e2476jW2H+VlkT4UgpoSJbqGQkBB0794dn376qRzQMnnyZFiC3NxcREREGJYjIyNll5gYkBQYGIj63m25bNkyrF27Vs43T0xMlOtdXV1hb2+P+m7OnDkYMWKErKecnBwZ644dO7Bp0yZz7xpZeNthye2GwLbjFqC3YJ9//rk+MDBQb2NjI6eJHThwQG8ptm/fLqdKVb1NnDhRX99VF5e4LV68WG8JpkyZom/cuLF8X3p7e+sHDx6s37x5s7l3i/4DbYcltxsC2w7z40+fExERkSIsckwFERERmR6TCiIiIlIEkwoiIiJSBJMKIiIiUgSTCiIiIlIEkwoiIiJShMUnFeKbxl5//fVb6hvHlMT46jdLj6++svR6YXz1W9EtHJ/Ff09Fdna2/DY18eMrLi4usDSMr36z9PjqK0uvF8ZXv2XfwvFZfE8FERERmQaTCiIiIqqfPygmflEtPj5e/tiLSqUySTdR5f8tDeOr30wdn7jaKX6IyN/fH1ZW9eecgu2Gshhf/ZZ9C7cbJh9TcfnyZQQEBJjyKYmoitjYWDRq1Aj1BdsNovrRbpi8p0KcaQi9l0+DxsEGlmhtq42wZGNadjD3LtAN0qIEe/C34TisL8r3N/pYEFyc6k8PS10kanNhyZ5s28vcu0AmaDdMnlSUd12KhELjaAtL5OJsmY1eOY3K2ty7QDfqSr+kKS4hKKl8f0VCYanHV57WMuMqx3bjv9FuWPa7mIiIiEyGSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQUREREpgkkFERERKYJJBRERESmCSQURERH9N5OKuxr2wS89X8aGfu9hYfCzaOUcWKvHDfTpjG0DP8ab7Scbre/r1QHvdXocq/u+Je9v5uQPs3J4ECrv7VD5hkLlsQKw7lhzWfuxsPK7YHQTjzNi5QmV63tQee+ByvcUVO4/AOrGMJfRTw3DT5e+wPr8X/DZ/nfQqlvza5bvN74nfjj7qSz/7cmP0H1El6vKTHzjPvwW9y3W5f2C9za/gobN/WAulh5fvab0sWU7FCr3xVD5HJL3Q9MG5uToOAm+fofg3zAS3j7rYW3duVaPs7e/Cw0bJcDDc7HReisrL7i5fwq/BsfRwP8SPL2WQa1pAnOx9GNrtIXEd0NJxRdffIGgoCDY2dmhR48eOHToEExhgE9nPNH8LvwYtQlPHPkYF3Pj8V6nx+Bm7XTNx/nauePxZqNxKvPiVffZqW0QmhWJ7y6ug9nZ3QGV81zocxdCn3o3oD0HlfsiwMqjxofodTnQJfcy3PQp/Y3uV7l9BagDoM94EvrUu4DSeKg8lgIqe5ha/3t74/GPJuLnN//Ak8Ev4tKpaMzf+BLcvF2qLd+2V0vMXTYTGxf9gye7zsbetYfw+urZCGoXYChz3+y7cPfTI7DgyW/xdM85KMwrwvyNL8Pa1hqmZunxKcFcbcfNOLbEMaQvPgp9zgcwN3v70XB1ex052R8hOWkYSorPwsv7V1hZeV7zcWp1I7i6voqiogNX3efpuRgaTWOkpU5CSvLtKNVehpfX71Cx7VBcfwuKr85JxfLlyzFr1iy89tprOHbsGDp16oRhw4YhOTkZN9v4gP74O/4ANiUeRnR+Ej4NX4EiXQmGN+he42OsoMLctg9hadQmJBSkXXX/1qSj+ClqM45mnIe5qRymAPnLgYKVQGkE9NmvAvoCwH78NR6lB3SplW6VYlQHQWXTpWw72tNAaWTZ37AD7O6EqY177k5s+H4bNi3ZgZhzl7HgiW9RlF+MYVMGVVt+zDMjcXjjCfzx4Z+ICYvD0leXI+LYJdw1Y3hFmWdH4pd5K7H/zyOIPB2D9yYuhKe/O/rc3Q2mZunx/VvmbDsUP7aEwrVA3kKgeB/Mzcn5ceTl/YL8/OXQas8jM3M29PoCODjef41HWcHd4wtkZ38IrTba6B6NpilsbEOQmfEiSkpOQqu9iMzMF6FS2cHefgxMzdKPrXEWFF+dk4qPP/4Yjz76KCZPnoy2bdvi66+/hoODAxYtWoSbSaNSo6VTIxyr9OGvhx7H0s+jrUtQjY97OGgoMotzsSHhIG5t1oB1O+iNGii9bLBU1ld3axmoHKDy3gGV966yXglNpS4zlc2VzRQbbxPFUNmEwJQ01hq0DG6KY1tPVeyJXi+X2/ZsWWM2fmxbRXnhyOaTaHOlvF8TH3g2cMfxracN9+dn5yPsYATa9moFU7L0+JRgrrbjphxbtxRrWFt3RFHh7krr9HLZxia4xkc5u8yCTpeG/Pxfq7m3rO3Q64uMtimWbWxrPom7GSz92NJYWHx1SiqKi4tx9OhRDBkypGIDVlZyef/+/dU+pqioCNnZ2Ua3G+Fq7Qi1lRoZxTlG6zNKcuBh61ztY9q7NsGIBj3wUfjvuOVZuUOl0pSdEVVWmgZYeVf/GO0l6LPmlF3ayHyhrF/G43fAyq/i/tI4qJyeB1SiG80acHwMKnWDmrd5k7h6OUOtUSMjKctofUZyFtz93Kp9jFifWbV8UiY8rpQv/1+sq1rG3bf6bd4slh7fv1XXtkOpduOmHVu3ECsrDxmfTpditL5UlwK12qfax9jYdIejw/3IzBCxXU2rjYBWexkurnOhUrnKtsPJeTo0moZQq31hSpZ+bLlaWHx1SipSU1NRWloKX1/jN5VYTkxMrPYx8+fPh6urq+EWEFBxzedmslfb4v/aPICPw39HdkkeLFLJCaBwjbw+jJJD0GdOB3TpUDlMuFJAC33GdEDTBFa+R8sGatr0gL5oBwCdmXee/kvq2naYq92o/bFVf6lUjnD3+BwZmf+DTpdeQykt0tOmyssg/g3D4N/wEmxt+6CwYBvbDromDW6yOXPmyOuo5cQZx400EFkleSjVlcLdxrhXwt3aGelFxr0Xgr+9JxrYe+LtDlMN61Qqlfx/c/8PMPHgu0govHqMhdnoMqDXawErL+P1ak+gyhlIzbSA9qzx7A7tGejTRkOvEoNZbQB9etnI95KKbjFTyErNQam2FO6+4qyngruPKzISjbPpcmK9W9Xyvm5Iv1K+/P/K68qXL56MgilZenymplS7cVOPrVuESAxEfFZVel3UVt4oLb16vIpGEwSNJhCenkuvOr/0bxiLpMS+KC2NRknJKTlAU6VyhkplIy+ViFklxcUnYUqWfmxlWVh8deqp8PLyglqtRlJSktF6seznV323oK2tLVxcXIxuN0KrL8X53Mvo4t7CsE4FlVw+m331ixSTn4yph97HY0c+Mtz2p57BicwI+XdKUfWVZT4lQMkZqGx6VVqnAmx6Q19yvJbbsAI0LQFdNQPf9LkyoZCNonV76IvEGYfpaEu0OH/0EroM7mCU5InlsweqHyR7dv95dBlUUV7oOqQjzl0pnxiZjLSEDHQZ3N5wv4OzPVr3aI6z+8NhSpYe379V17ZDqXbDJMeW2ZXIBMDWrm+ldSrY2vZFcfHRq0uXRCApcQCSk4YYboWFm1FUtFf+XVoab1Rer8+RCYWYTmpt3QmFBZtgSpZ+bGktLL46JRU2NjYIDg7Gtm0VH0g6nU4u9+pV+YC9OVbE7sTIBj0x1C8EgQ4+mNlyvJwSuimhbFrai23ux9SmI+XfJTotovISjW652gIUaIvk3yJJEZw1DvK7KRo7lDVsAQ4+crlqj4gp6PMXAQ73AXZjAHUzqFzeLJv6KUasizea6/tl4yPKOc4AbPrKKaPQtIXK9SNA3RD6/D8qytgOB2y6l5WxHQyVxxKgaCtQvMfk8a38ZB3umDYYtz/SH4GtG+KZrx6FnaMtNi3eLu+fvWQGprzzgKH86s/Wo9vwzhg/604EtPLHw6/dg5YhzbB24caKMgvW44GXxqHXqBAEtQ/E7KUzkBafgb1rDjO+W4i5246bcmyJsQbiuynUVwZwiu9wEMtVe0RMIDfnGzg6PggHh3ug0bSAm9t7UFk5ID/vN3m/u/tncHGZe6V0EbTacKObTpcFvT5P/i2TMDFHzP5O2Nj2glodCDu7YfDyWo7Cgo0oKtpp8vgs/dhaaUHx1fnyh+iSnDhxIkJCQtC9e3d8+umnyMvLkyO6b7YdySfgau2ESU2Gw93GBRdz4/B/p75FRkmuvN/H1l2Omq2L3l7tMLtNxbSrV9o9Iv9fGrlJfh+GSRX+Db0YdOX8bNkAspJz0GdMrZjKphZfzFURn8rKBXB9u6ysLkuejenT7pNT5gzUPlA5zpVfgiW7egvWQJ/7Bcxh5+/75Lxr8YUsYqDRxRNRmDtiHjKTywYc+QR6Qa/TG2Xj8x9cgElv3Y/J8x5A3IUEvD7mfUSdiTWUWf7+Wtg52mHmN4/Dyc0BoXvCMGfEPJQUlTWMjO/WYc6246YcW3aDYeX6nmHRym2B/F+f+xn0uZ/DlAoK/oRVpiecXWZDrfZGSckZpKY+AN2VwalqTUPo6zgWQgzIdHV9XW5PXEbJz/8DOdmf3KQI/tvH1k4Lik+lr+unMICFCxfigw8+kAOsOnfujM8++0x+kU1tiGujYuBVv7+egsbRFpZoS5u/YMmG+dfum/ro1qPVl2AH1iIrK+tfXlK4MTfadpS3Gxnnm8LFud59EXCtJGjLTo4s1aTAypdnyFLbjRsaqDljxgx5IyKqC7YdRJbNMlN+IiIiMjkmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCA3MJDbVDVb5drBE7bY+BUvmuSkBls5+WKS5d4Gq0ffYOKgdbGGJrDWlsGTaPy3/HNZ7dDj+6yy/lomIiMgkmFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEiNKhnHmoegmmtesHbzgnnMpPw5vGNOJUeX23ZoQ1b48k2fdDYyQMaKytE5aRj0fkDWBN92qhcM2cvzO44GN29A6G2skJEdiqm7/sDCfnZMLUHenTClNuC4eXkiLDEFMxbtx2nLydVW/aekPYY3aUtWvh6yuWzccn4ZMueGsu/dtdgTOjeEfPX78CP+47DHMY26oX7g/rDw8YZF3MT8EnYWpzLjr3u4wb7dsIbHR/EruRQzD35o9F9U5sNxaiG3eGsscfpzCh8GLYal/NTYQ6jnxqGe14YDQ8/N1w8GY0vnlmE8MMRNZbvN74nJr45AX5B3oi7kIjv/+9nHNpgXDcT37gPI6YNhpObI87sDcNnT32HuIhEE0RjWe4L6o6JzfrCy9YJ57MT8W7oeoRmxlVbdrBfW0xt0Q8Bjh6wVqkRnZeGny7txbrLJw1lnmg5EMMbdoCfnStKdKU4mxWPhWFbcTrzMszhnsCeeLjJbfC0dcKFnER8cPYvnMmqfl8G+rbD5Gb9EeDgCY1KjZj8VPwSuQd/x5+otvycdndhXGAPfHRuHX6N2gdzuK9xD1l/nlfq770z62qsv0Gi/pr3R6CjR1l8eWn48dJerI+riO+JloMwzL9K/YVvQaiZ6m+0hbQdde6p2LVrF0aNGgV/f3+oVCqsWbMGpnJHQFvM7XQ7Pj+zC3dt+Q5hmUlY3O8BeNg6VFs+q7gAX57bg3u2Lcadm77FyqiTeLfbaNzm29RQJtDRHb8NmoiLOal4cMdPstwXZ3ejqFQLUxvRoSVevKMfvvjnAMZ98QvCE1Px3aSx8HC0r7Z8tyaN8PepMEz6YQXu//o3JGTl4PtJY+Hj4nhV2SFtm6FTgB+SsnNhLoN8O2FGq1FYfGkrph5cgIicBHzcdSrcrK/e38r87NwxveVInMi4dNV9DwYNwPiAPvjw3Co8duhzFJQW4+MuU2FjZfp8uf+9vfH4RxPx85t/4MngF3HpVDTmb3wJbt4u1ZZv26sl5i6biY2L/sGTXWdj79pDeH31bAS1CzCUuW/2Xbj76RFY8OS3eLrnHBTmFWH+xpdhbWuN+sSc7YYwzL89Xmg7At+c344Ju75CeHYivuoxER421b/3skry8f2FnXhkz3cYv3Mh1sYewxudxqC3d3NDGZFozD+9DuN2LsSkvd8jPj8DX/WcCHeb6tujm+l2vw54rs0d+C5iGx7a9wXOZyfg826T4V5DfNkl+Vh0cQcm7/8aE/Z+hr8uH8OrHcahp1eLq8oO8G2L9m4BSC7MgrkMbdAez1+pv/t3fymTii+7T7pGfAX4PmIHHtn7Le7ZtRBrL5fVX6/K9ZebindD12H8rs8xed93iC/IwFc9Jpml/vpbUNtR56QiLy8PnTp1whdffAFTm9KyJ5ZfOi6TA9Gb8MrR9SjQluCeJp2rLX8wJRpb4sJlwhCTl4GlFw4hPCsJwd6BhjKzOgzEzoQIvH9qG85mJspy2+LPI70oH6Y2sU9X/HEkFKuPncXFlHS8vnYrCku0GBvcvtrys//YiF8PnkJYQgoiUzPwyuotsFKp0KtpRXyCSDJeunMgZv++EdrSUpjLhMa34a/LB/F3/BFE5SXjg3OrUFhagjsbdqvxMVZQ4dUO9+OHi1sQX5B+1f33BPbFj5HbsCflLC7mJuLtM8vhaeuC27zbwdTGPXcnNny/DZuW7EDMuctY8MS3KMovxrApg6otP+aZkTi88QT++PBPxITFYemryxFx7BLumjG8osyzI/HLvJXY/+cRRJ6OwXsTF8LT3x197q75NbsVmbPdEB5u2hurYo5gbexxXMpNwdun/pLvvbsDu1Zb/khaFP5JPIfI3BRczs/AssgDuJCThC4ejQ1lNsSdwsHUS4jLz8DF3GR8eHYjnK3t0MLFD6b2YJO+WBN7GH/FHUNkbjLmn1mLwtJijG4UXG35o+mR2JF0FlF5KYjLT8dv0fsQkZOIzu4V8Qneti74X9tReOXk79DqdDCXh5v2warYIzI5kPV3+k8U6kpwd0D18R1Ji8R2Q/2lY1nk/rL6qxTfhnhRfxcN9ffR2Q1l9eds+vobZ0FtR52TihEjRuDtt9/GmDFjYErWVlZo794Ae5MiDev0APYlR6KLZ6NabaOXTxCaOHvicEq0XFaJLLxBc0Tmpssej4OjZ2HF4CkY4t8KpmattkI7f1/sj4gxrNPrIZc7Bzao1TbsrDXQqNXIKig0rFOpgPfGD8ei3UcRkZwGcxFdkC2dG+JIekV3nh56HEm/gHauxg1ZZZOaDkFmcS7Wxx++6j5/ew942brgcNoFw7o8bSHOZseivVvN27wZNNYatAxuimNbTxnW6fV6udy2Z8sazzaObasoLxzZfBJtrpT3a+IDzwbuOL614nJdfnY+wg5GoG0v079H/w1ztRvl7702rv44kHrJ6L13IPUiOrpXnNldS3evpghy9MLRtKgan2NcYIg8QxZn0aYknru1iz8OphofW4dEfG7GJxg16ebZDI0dvXE8vSI+FVR4s9M9+OnSblzKTYa5lNffwZSLRvGJ5VrXn2dZ/R2rFF919Zdjjvqztqy246b3ERcVFclbuezsGxunILqkxLiItCLj7vvUwjw0dfaq8XFO1rbYe+dM2KjV0On1eO3Y34bExNPOUd7/eOve+CR0h+yt6OfXDF/2uQcP7fgRh1IqPuBvNjcHe2jUVkjLNe4hEctNvN1rtY0Xht+G5Oxc7LtYsd/TbuuGUp0eP+03zxiKcq42jtBYqZFenGO0Pr04F40dfap9TEe3INmLMfnAp9XeL8ZlCBnFxu+JjKIcw32m4urlDLVGjYwk4y7ijOQsBLRuWO1j3P3ckFm1fFKmvKYqlP8v1lUt4+5bdp+lUqrdqGg71Fe1HWK5idM12g6NLbbc/j9YW2mg0+vwzul1MhGprJ9PS7wXfC/s1NZILczFE/uXIrPYtL2cblfiE8dSZWI5yMm7xsc5amyxYeD/yUuFpXod3jv7Jw6mVSQmE5v2k+tFL4Y51Vh/Mr5r19/mIbMr6i/0r6vq7zafVniv65X6K8rFEweWILPEtPXnamFtx01PKubPn4833ngD5pJXUoTRW76Fg8YGvX2aYG6noYjNzZSXRkTXurA17jwWnz8o/xaDP7t6BuD+ZsEmTSr+rWn9umFEh1aY+P0fKNaWXeJo6++Dh3t3keMz6ht7tS1ebj8B759dKa9v03+LudsNIU9bjHt3finbjh5eTfF8u+GyK11cGil3OC1SlhEf7OMah+CDkPvw0O5vkF6ch1tdvrYYD+z9HA5qW9lT8VzrO+SlEHFpRPR8TAjqjYf2LkR9Jervvl1fyPrr7tVMjqkRlzrEpZFyh9MuyTKi/sYGdsP7wRPw0J6vkVEP6u9WddOTijlz5mDWrFlGZxwBAbXrsqosozhfXtMTI38r87JzlGcINRGXSKJzMwwJQzMXLzzRpo9MKsQ2xajfiOwUo8dE5KQixKvu+/hvZOYXQFuqg6eT8SAhsZxapfeiqsl9g/FovxBMWbwK55MqZj2EBDWEp6MD/vnfNMM60Rsye0Q/PNK7C4Z8uAimklWcB62u9KoeBA8bJ6QVGfdeCA3tPeTljXc7TzKsE+NFhB2D5+OBfR8Yej3cxTYq9YC42zojIqf6GUE3S1ZqDkq1pXD3dTVa7+7jioxE47OFcmK9W9Xyvm5Iv1K+/P/K68qXL56svhvXUijVblS0HaVXtR1iWZyd1kR0scfml43jEQM7mzh5Y2rzfkZJRUFpiSwjbmLWx58DZ+LuwGAsitgFUxE9I2XHllOtjq3K8YkkSTifkyDjm9S0v0wqungEyUGs6wbMNpQXvQUzW9+B+xv3weidH8BUaqw/m7rX35Rm/YySisJq6m9MQDAWXTRd/WVZWNtx07+nwtbWFi4uLka3G1Gi0yE0IwG9fYMM68RHjOh9OJ5W+ylAYuS5jZXasM3T6fFo6lw2JbNcEycPxOWZdqRzSakOZ+KT0LNZRcMpPkPF8omYhBofN/W2EDw5sAceW7oaZ+KMp5L+efwc7v78J4xd+LPhJmZ/iPEV05ashilp9aU4nxOHYI/mRtdsxfKZrLIxLpXF5Kfg4X0fyUsf5TcxGPNY+kX5txiJLgZuphZlI8SzYsS6OOtq6xKA0Myrt3kzaUu0OH/0EroM7mD0XhPLZw+cr/YxZ/efR5dBFeWFrkM64tyV8omRyUhLyECXwRUDdR2c7dG6R3Oc3R8OS6ZUu1H+3juXFS97Gyq/98TyqYzrT2eunNSKrvTrlSlvX0xFxBeWHY/unsbHVjevZjiVWfve1rJ9L4vv77jjuH/P53hw70LDTRxzYnzF00cWw5TK60+Ma6kcX/e61h9UsFFfu/5UsLpuGaVpLaztqFffUyG+Y+KD7nfhdHqC/G6KSS27w15jjRWRZXPHxX1JBTn48PQ/cvmJ1n1wOiMeMbkZ8mARgzLvbtwBrx3927DN78L3Y0HPcTicGoMDyVFyTMUg/5Z4cIfxdyGYwtK9xzB/3DCExiXj9OVE2Ztgb2ON1UfPyPvfHT9MJgWfbN4rl6fdFoKnh/TCC79vQFxGNryu9HLkF5fIW2ZBobxVJmZ/pObmISq1rPfGlH6L3o2X2t2LsOzL8rsp7g3sC3u1DdbHH5H3v9zuPqQUZeGbiI0o1mkRmWecJOVqy2KpvP6PmD2Y2GQQYvNTkVCQjmnNhiKtKBu7U8peM1Na+ck6zF4yHeePXET4oQiMmTkSdo622LR4u7x/9pIZSI1Px6K5y+Ty6s/W46Mdb2D8rDtxcP0xDJjQBy1DmuHTx78xbHP1gvV44KVxch56QmQyJr15H9LiM7B3zdUDV6lmP13ah7c6j8WZzDj53QYPNe0l33trYo7J+9/uPA7Jhdn4LGyLXJ7SvB/OZsbJM1jRdtzm0wIjG3XGvNN/yfvt1daY1qI/diSGIbUoB242jpgQ1B0+ds7YEm/69574jonXO47H2ezLOJN5GQ8E9ZHxiamiwhsdx8v4vji/WS6LHolzWXG4nJ8mE6U+3q1wh38XOWtEyCopkLfKRE+x6BGMzjP9d8CI7wh5q/M4+V0S4nskHmzSW8a3NvaovF/cJ+L7vLz+mvXD2ayK+uvr01LW3zun/5T3izEUjzYfgB1J52Rvh7j8Ib4Ho6z+Qk0e30oLajvqnFTk5uYiIqJiME9kZCROnDgBDw8PBAbWbqTxjfo79iw8bR0ws31/+eVXZzOTMGXXMqQVlV3/8ndwkYMxy4mE442uI+Bn74LCUi0u5aTi+YNr5HbKiSmnrx5bLxOQVzoPw6WcNMzY9weOptY+A1bKhtPn4e5oj2cG94KXswPOJaTgsSWrkZZXdvmjgauzUXwTenSEjUaDzx4YZbSdhdv2y++6uNX8k3RSNr7ig9/jyiWK54/9YBho6WvnBp28YFV7v0TtgJ3aBrPbjIOTxk5++dXzx3+QSYmp7fx9n5xXLr5wRgykungiCnNHzENmclmvl0+gF/Q6vdHZxvwHF2DSW/dj8rwHEHchAa+PeR9RZyree8vfXws7RzvM/OZxOLk5IHRPGOaMmIeSohLUJ+ZsN4RN8aHyOw2eajVYfvlVeHYCnjr4o2Hsg5+9K3SomDIpkoa5HUbB194FRaUliMxNxUvHV8jtCKV6vexOHx3SRX4gicF9ImGZvPcHOT3R1LYknpbxPdFiCDxtneX3VDx9eLFh8KafOLYqt41qG7zYbjR87FxlfGJqqZg2KrZzK9qcEAp3W0c82bJS/R1aaqi/BvZucsZEOXuNjaw/Q3yy/v6Q2xHEayEGeX4U8ADcrCvqb8q+781SfzstqO1Q6SvXRC3s2LEDAwcOvGr9xIkTsWTJkus+XlwbdXV1RZPFc2HlYAdLZH3S+NqfpfEcUPPlGEthP6ziuqsl0epLsANrkZWV9a8uKdSVUu1Gu9/+B7WDLSyRtcZ83yFjCtpSy/9VCO/RlnlZsi7tRp17KgYMGGCUERIRXQ/bDaL/BstPHYmIiMgkmFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSKYVBAREZEimFQQERGRIphUEBERkSI0MBPHPY5Q29jBEuX762HJCpb7wdLF/+IGS6TLLwSmrUV9lXfBDVZ2ltlu6NWW3W5Ap4Kly/q4JyyRrrAQmFO7doM9FURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIDeqZ+/p1wsTbQ+Dp4ojzl1Pw3u/bERqdWG3ZsX064M4ebdDc30sun41JwsK1e68q/+SdvTG2T3s429vhxKU4vPPrNsSkZMIcHgzuhKm9QuDt5IiwpBS8tWk7TsVXH9+9XTrg7g5t0MK7LL4ziUn4ePveq8o38/TAC4NvQ/fARlBbWeFiahpmrPgLCdk5MLV7BnbCI8NC4OnqiAuxKXj/1+04E1l9fE39PfHEXb3RprEP/L1c8eFv2/Hr1uNGZRxsrfHk3X0wsGtzuDs7IDwmWZY7G5UEc3i4RTAebd0D3vZOOJeRhNePbsap9IRqyw5r1ApPte2Nxs7u0FhZISonA9+HHcSaqFBDmfd73InxTTsaPW5nwkVM3rH8psdiaR7q3AmPdguBt6MjzqWk4I1t23Eqsfr33n0dOmBMuzZo6VV2bIUmJeHD3XuNyr8/fBjGtW9n9LhdkVGYvHIVzOHhTp3xaHBFfK9v/wenkmqIr30HjG3bFi09r8SXnIQP9uwxKv/+0GEY36690eN2RkVi8mozxSfqL6RSfP9cu/7GtjWuvw/2VKm/YcMwvkr97RT1t8pM8XXsjMdEfA6OOJdaVn8na6i/CaL+2lTU3+nkJHy4d49R+Q9E/bW9uv4mrVl16yQV8+fPx6pVqxAWFgZ7e3v07t0b7733Hlq1agVTGBrcEs+P6495v27D6agEPDioK758eizuen0xMnILriof0qIRNh4Jx8lL21FUosXkod3w1dNjMe6tH5GclSvLTLq9Gx4Y0Bmv/LgJcWlZeOrO3nKbY99cimJtKUzpjrYtMef2/nh1wzacjEvApO5d8cP9YzHsq8VIz786vu6NG2HdmXAcv7wdRVotHu3dDYseGIuR3/yIpJyy+ALcXbFs4n1YcSIUn+3ch9ziYrTw8pTlTe32bi0x697+eOfnbQi9lIAHhnTFwpljMfblxcjIuTo+OxsN4lKysPXIeTx/X/9qt/nKpKFo5u+JV77fgJSsPNzRsw2+mjUe419dipTMstfAVEYGtsHcLoPxyuGNOJEWj8mtumHpwAkYsu4bpBXlX1U+s7gAX5zdi4vZaSjRlWKQfwuZRKQV5mF3YqSh3I74i5h9cJ1hubjUtO9LJZi77RjZqiXmDuiPV7Zuw8mEBEzu2hVLxo/F7YsWI62aY6tHQCP8FRaOY3HbUVSqxePdu2Hp+LEYvuRHJOVWvK92RkZi9oZNZq+bkS1bYW6//nhl21acSBTxBWPp2HEYsmQR0gqujq9nowD8FRaGownxKNKW4olu3fDj2HEY9uNSJOVVxLdDxLd5o/njE/XXv6z+Toj6C+6KpePGYoiov+riu1J/R+PL6k/GN24shi01rj8Z38Zbo/5e6tcfL/9TVn9TugRj6ZhxGLy0+vrr0SgAf4ZXqr+QsvobWrX+oiLxPxPXX50uf+zcuRPTp0/HgQMHsGXLFpSUlGDo0KHIy8uDKTw8KBir9oZi7YEzuJSYjrd/3YrCYi3u7m2cjZWbu2QDft91EuGXUxCVlIE3ft4ClUqF7q0DDGUeHNQF3208iB2nLuJCXCpeWboR3q5OGNipOUxtco9g/H48FKtOnsHF1HS8+vdWFJZoMb5z9fG9sGYDlh09iXNJKbiUloGX1m2BlUqFXkEV8c0a0Ae7Lkbig392y3KxGVn458KlapOUm+2h24Oxenco/tp7BpEJ6Xjn57L6u6tv9fGJ3oYFK3Zh8+HwahM8W2sNBnVtgc9W7MbxC3G4nJyJb//cj9iUTIwfYHx2bwpTW3XH8osnsCLyFCKyU/Hy4Q0o0GpxT9NO1ZY/mByDzZfPy6QiJjcTS84fRlhmMkK8K+pPKNZpkVqYZ7hllxSivjF32zElJBjLT4diZegZRKSl4+UtW1Egjq321b/3Zv29Ab+cOCnPiC+lZ2DOprK2o3dglbrRliI1P99wyy4qgjlM7RqM5aGnseLsGUSkp+PlrVtQoC3BPe07VFv+uY1/4+dTV+LLSMf/bdl8Jb5Ao3LiQ+iWiC+4rP5WnLkS35X6u6dD9fX33N8b8PPJivr7v8011N8tEt+0KvX30rYr9deulvW3dTNUUKHPLVB/deqp2LixIuMRlixZAh8fHxw9ehT9+vXDzaRRW6FNoC8WbT5kWKfXAwfDotGxSYNabUOc+WrUamTllTXKDT1dZQJxMCzGUCa3sBinoxLRqWkDbDoaDlOxtrJCuwa++GZvpfgA7IuKRueGtYvP3loDjZUamQVl8akA9G/eFN/vPyx7PNr6+eByZpZ8jq3nL8KURP21buyLxX8b19+hc9Ho0LR28VWltlLJ7YpeqMqKirXo3KIhTEnUX3uPBvjq7H6j+tubFIkuXrXbl96+QWjq4oH3T1a8H4WePo1xaMyzyC4uxP6kaHx0aqfs5ahPzNl2yLrx9cXXB6scWzHR6OJfy2NLo4G1OLYKC6/q0Tj01BPIKizE/phYfLxn71Vlbrby+L46bBzf3pgYdGlQh/jUVjKOyno2aoRDjz+JbBFfbAw+2mfG+A5VjS+6bvFZqauP78knyuKLicVHe80Un48vvqym/rrWsf4yq4nv8GNl9bfvsmnq71+NqcjKypL/e3h41FimqKhI3splZ2ff0HO5O9nLD5C0bONu5LScfAT51vz8lc0ccxtSsnINSYSXq0PZNqpsMz07T47ZMCV3B3t5XT01z3hfUnPz0dSzdvG9MOg2JOfmYl9kWXyejg5wsrXBY72749Mde/HhP7txW7MgLLxnNB7+6Q8cjrkMU3Grqf6y8xHkV7v4qsovKsHJiHhMG9VT9nykZ+djWI/W6NCsAWKTTTsmxt3Woaz+Co3PvMVyM2fPGh/nbG2LfXc9DRu1Gjq9Hq8e2Yg9iVGG+3clXMKmy+G4nJuJQCd3vNBpABYPuA/jtiyV5eur67UdSrUbgrt9DcdWXj6aXqPtqmx2/9tkt/Le6Bij8RObLlxAbFY2Gru54vnb+mLRuLEYv+xXk9aNIb78Ku+9/Hw0c69dfC/e1g9JuXnYExNtWLcrKgqbIiJwOSsLgW5ueKFPXyweMxbjfjNTfFXrT8RXy/p7sV9Z/e2pXH8yvgu4nJWNQDdXvNC3LxaPHYtxv95C9edRy/j6Xl1/O6/UX+yV+vtf775YcvdYjF1+c+O74aRCp9Nh5syZ6NOnD9rX0IVYfi31jTfegLmJ8RTDgltj2qe/m3yshCk81rsbRrZrjYd/+t1w3UxcChG2nb+IJYeOyb/FJZAujfxxf3BHkyYVN8urP2zAq5OGYdNHj0NbqkNYTDI2HQqXgzvrg9ySIty58Qc4aKzR2y8IL3UZIi+FiEsjwrqYs4ay4Vkp8vLIztFPyd6LfUkVyUd9Upu241ZpNwQxnuLOVq3xwPKKY0tYF17Rk3k+NRVhKanY8ehUeT1/X0ws6osnunXHna1a4YE/qsR3viK+8LRUhKWmYOeUaXI8xr5Y4960W9kT5fX3e831F36l/nZOq4f1F9Ido1q1wv0rrlN/KSnYZYL6u+EppeL6aGhoKH777bdrlpszZ448Kym/xcbeWGWJgZjiQ8PTpax3oZynswNSs699XfaRIcGYMrQbnvx8pRw3US41qyzzrbpNDxdHpF1nm0rLyC+AVqeDl6Pxvng5OSAl99r7MqVnsEwqpixbifDkVKNtlpSWIiI1zai8GK/RwMUZppRZU/25OCA168Zf68spWXjsg9/R56nPMHL2d5g4b5nsEREDPE0poyi/rP7sjHu4xHJKld6LysT5QnRuBs5lJuOHsEPYEBuGJ9v2rrF8bF4m0grz0djJHfVVbdoOpdoNIaOghmPL0QEp1xnTMS0kWH4oTVqxUn7wXIs4I0zLz0djNzeYkiE+hyrvPQcHpFQ5+61qWnCIHOQ3cdVKhN3q8VWtP4da1l+3bpi4sh7E5+BY5/ge7RqCJ7t1wyO1qb9s08R3Q0nFjBkzsG7dOmzfvh2NGjW6ZllbW1u4uLgY3W6E+EA6F5OE7q0qBqKIE3GxfCqy+il7wqTbQ/DoiJ54auFqOaW0MjHbQ1wOqbxNRzsbdAjyw8lLNW/zZijR6XAmIQm9mlSKD0CvoECciKt5X6b1CsH0vj0x9dfVCE1Iumqbp+OT0NTT+AOoiYc74rNMO51U9iJEJ6FbG+P669Y6EKcVeK3FgE+RnDg72KJXu8bYccK0Y0bEax2aniB7GyrXnxgncTw1rtbbEb1LNlbqGu/3s3eGu609kgtNO7NFKbVtO5RqNwx1k5RkNAhRHluBgTgeX/N777FuIZjRqycmr1yN00nXn6Ls5+Qku7KTTTT49Kr4AozjE8vHE64RX0g3PN2jJyatXlU/4qtSf2L5mvF1C8HTPXti0qp6UH/JSehTTf0du0Z8jwd3w4wePTFR1F9y7eO7XqJi0ssfer0eTz/9NFavXo0dO3agSZMmMKWf/jmKtx4ZjrPRSfK7Jh4c2BX2ttZYu/+MvP+ticORnJmLz9fuMUwXferOXpizeAPi07MMZ8niWnxBUYn8+5d/juPRET0Qk5yBuLRsTB/VWyYa209GwNQWHzyK90YPl8nBqbhETOzRFfbW1lh5siy+90cPl1NFP9peFt+jvbrh2f69MGvNBsRlZhky+fziEuSXlMX3w4Ej+GTsSByOicOBqFj0axaEgS2bysskpvbzlqN4Y8pwnBP1F5kop5SK+vtzb1l84j4xDXThqrL4RI+D+K4KwVqjho+bM1oGeMv6EzM9BJFAiEMwOikdAT5ueHZ8P0QlZMgZJqb2Q/ghfNhzFE6nJ+CknFLaXV7WELNBBHFfUkEOPji5Qy4/2baXLBudkynHVAzwb4a7g9rLKamCeOwz7W/Dxtgw2dsheide7DwQ0Tnp2J1wCfWJuduORUeO4oMRw+WHy8mERDkl0cHaGitCy94nH44YjsTcXHy4u+y991j3bpjZuxeeW79BjikQZ42COK7ETTz2md69sPH8BdlIizEVL/brh+iMTOyOqriubSo/HDuKD4cNx+nkRJxMTMTkLlfiO1P2nSfiPjGV8oO9ZfE9HtINM3v1xnMb/sbl7Bri69kLGy9ckL0djV3d5LiL6MwM7I42/WW3H44exYfDh+N0YlJZfF2r1N/wK/HtuRJftyv19/c16q/Xlfhugfr7/thRfDR0uPyeEBHflPL4zpbVn7gvMc+4/p7r2RszN9Zcf8/26IUNERX19399y+pv102uP01duy2XLVuGtWvXwtnZGYlXvkjE1dVVzj2/2TYfPQ93Jwf5ZVVeLg5yquhTC1chPafsMkYDd2fodRUDUO7t1xE21hp89Ngoo+18vX6/vAlLthyWH2yvPHC7PMs9fjFObtMc4y7+PnseHg4OeKZ/b3g7OsjxD1N/XYW0KwOUGrg6Gw2wEeMibDQaLBxvHN/nu/bLm7AlPAKv/b0Vj/fpjpeHDkRkWjqeXvEXjsbGmzg6YMvhsvoTX2glErzzsSl4+tNVcoCl4OfpLD98ynm7OeHX1x42LD8yPETejoTH4vEP/pDrnOxtMWNsX/i4OyE7rxDbjkXgy9V7ZM+Iqa2POQcPWwc816GfvOwhvvxq0o7lhsGb/g4uRvVnr7bBmyHDZe9DYalWTi2dtf9PuR2hVK9HazcfjG3SAS7WdkguyJHfX/HJ6V0o1tWvcUHmbjvWh5cdWzP79JYNsJiKN3nFKtkdLIjLgZXr5sFOHWGr0eDLu4yPrQX79uOzfftl3bTy8sLYdm3hbGsrB0jviYrGx3v3meW7DtafD4eHvT2e69XHEN+k1SvlYD/B39n4vfdgx05l8Y0abRzf/n1YcGA/SnV6tPbyxti27eByJb7dMdH4ZN9e88Qn6s/eAc9Vqr9JK1dVxFdT/Y2+uv4W7C+rv9beZfVniC86Gp+Ysf487e0xq7z+UlMwaU2l+nNxgU5eLC3z0JX6++pO4/r79ECl+vOuVH95ZfF9vP/m159KX7kVv17hKwP/qlq8eDEmTZpUq22IUdyiIWk/dR7UNnawRPn+sGjO9XN8YJ1kDK5/3wVRG7r8QkRPe0uOU/g3lxTq6t+2HeXtRtBb82BlZ5nthl5df2fz1Iqu+veAJdFbaB3qCgsRM+flWrUbdb78QURUV2w7iP4b+INiREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIJhVERESkCCYVREREpAgmFURERKQIDUxMr9fL/0uLC2GpSi03NKm0GBZPl2+ZlagrKDI6DuuL8v3VFVpmvQh6df2qkzrTqWDp9BZah+XHXW3aDZXexK3L5cuXERAQYMqnJKIqYmNj0ahRI9QXbDeI6ke7YfKkQqfTIT4+Hs7OzlCpbn7mmp2dLRsj8WK4uLjA0jC++s3U8YnDPScnB/7+/rCyqj9XP9luKIvx1W/Zt3C7YfLLH2KHzHGGJF54S3xzlWN89Zsp43N1dUV9w3bj5mB89ZvLLdhu1J9TFSIiIrqlMakgIiIiRVh8UmFra4vXXntN/m+JGF/9Zunx1VeWXi+Mr36zvYXjM/lATSIiIrJMFt9TQURERKbBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIgUwaSCiIiIFMGkgoiIiBTBpIKIiIighP8H8uZwOSy++AgAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 364
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Transformer-Block",
   "id": "4f45a6b4b60bcb8a"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "## 核心原理总结\n",
    "### 模块化设计：\n",
    "\n",
    "每个Transformer块包含三个核心子层：自注意力、可选交叉注意力和前馈网络\n",
    "\n",
    "各子层遵循相同模式：子层处理 → Dropout → 残差连接 → 层归一化\n",
    "\n",
    "### 注意力机制：\n",
    "\n",
    "自注意力：建立输入序列内部各位置的关系（编码器捕捉语法语义，解码器建立目标序列关系）\n",
    "\n",
    "交叉注意力：连接编码器-解码器，帮助解码器关注相关源语言信息（机器翻译等任务关键）\n",
    "\n",
    "### 残差连接：\n",
    "\n",
    "每个子层输出与输入相加（x + Sublayer(x)），确保梯度有效回传，缓解深层网络梯度消失\n",
    "\n",
    "允许构建更深的网络结构（如12层、24层的Transformer）\n",
    "\n",
    "### 层归一化：\n",
    "\n",
    "在残差连接之后应用，稳定各层输出的分布\n",
    "\n",
    "使用可学习的缩放和平移参数，增强模型表达能力\n",
    "\n",
    "### 前馈网络：\n",
    "\n",
    "通过扩展-压缩维度（如512→2048→512）引入非线性\n",
    "\n",
    "增强模型对特征的重组和抽象能力\n",
    "\n",
    "### 注意力掩码：\n",
    "\n",
    "自注意力掩码：防止解码器查看未来信息（auto-regressive特性）\n",
    "\n",
    "交叉注意力掩码：通常用于屏蔽源语言序列中的padding位置"
   ],
   "id": "2ebe3c155227669c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.822182Z",
     "start_time": "2025-03-24T10:58:40.815534Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 通过 @dataclass 装饰器自动生成 __init__(), __repr__() 和 __eq__() 等方法\n",
    "@dataclass\n",
    "class TransformerBlockOutput:\n",
    "    \"\"\"\n",
    "    Transformer块的前向传播输出结果容器\n",
    "    \n",
    "    属性说明：\n",
    "    hidden_states: Tensor\n",
    "        经过当前Transformer块处理后的隐藏状态，是各子层处理结果的最终聚合\n",
    "    self_attn_scores: Tensor\n",
    "        自注意力机制计算得到的注意力权重矩阵，反映输入序列各位置间的关联强度\n",
    "    cross_attn_scores: Optional[Tensor] = None\n",
    "        交叉注意力机制计算得到的注意力权重矩阵（当存在时），反映编码器-解码器位置间的关联\n",
    "        Optional 表示该字段可能不存在（如在纯编码器结构中）\n",
    "    \"\"\"\n",
    "    hidden_states: Tensor\n",
    "    self_attn_scores: Tensor\n",
    "    cross_attn_scores: Optional[Tensor] = None\n",
    "\n",
    "\n",
    "class TransformerBlock(nn.Module):\n",
    "    \"\"\"Transformer模型的核心组件块，包含自注意力、交叉注意力（可选）和前馈神经网络\"\"\"\n",
    "\n",
    "    def __init__(self, config, add_cross_attention=False):\n",
    "        \"\"\"\n",
    "        参数说明：\n",
    "        config: dict\n",
    "            模型配置字典，包含以下关键参数：\n",
    "            - d_model: 隐藏层维度\n",
    "            - num_heads: 注意力头数\n",
    "            - dropout: dropout概率\n",
    "            - dim_feedforward: FFN中间层维度\n",
    "            - layer_norm_eps: 层归一化的epsilon值\n",
    "        add_cross_attention: bool\n",
    "            是否添加交叉注意力机制（用于解码器层）\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        # 初始化基础参数\n",
    "        self.hidden_size = config[\"d_model\"]  # 特征维度（如512）\n",
    "        self.num_heads = config[\"num_heads\"]  # 注意力头数（如8）\n",
    "        dropout_rate = config[\"dropout\"]  # dropout概率（如0.1）\n",
    "        ffn_dim = config[\"dim_feedforward\"]  # FFN中间层维度（如2048）\n",
    "        eps = config[\"layer_norm_eps\"]  # 层归一化的小常数（如1e-5）\n",
    "\n",
    "        # ========== 自注意力子层 ==========\n",
    "        # 多头自注意力机制：计算输入序列自身的注意力关系\n",
    "        self.self_atten = MultiHeadAttention(config)\n",
    "        # 层归一化：稳定训练，加速收敛\n",
    "        self.self_ln = nn.LayerNorm(self.hidden_size, eps=eps)\n",
    "        # Dropout：防止过拟合\n",
    "        self.self_dropout = nn.Dropout(dropout_rate)\n",
    "\n",
    "        # ========== 交叉注意力子层 ==========\n",
    "        # 仅在解码器需要连接编码器输出时启用\n",
    "        if add_cross_attention:\n",
    "            # 多头交叉注意力：计算解码器查询与编码器键值的注意力\n",
    "            self.cross_atten = MultiHeadAttention(config)\n",
    "            self.cross_ln = nn.LayerNorm(self.hidden_size, eps=eps)\n",
    "            self.cross_dropout = nn.Dropout(dropout_rate)\n",
    "        else:\n",
    "            self.cross_atten = None  # 编码器层不需要此模块\n",
    "\n",
    "        # ========== 前馈神经网络子层 ==========\n",
    "        # FFN结构：通过非线性变换增强模型表达能力\n",
    "        self.ffn = nn.Sequential(\n",
    "            nn.Linear(self.hidden_size, ffn_dim),  # 扩展维度（如512->2048）\n",
    "            nn.ReLU(),  # 非线性激活\n",
    "            nn.Linear(ffn_dim, self.hidden_size),  # 恢复原始维度（2048->512）\n",
    "        )\n",
    "        self.ffn_ln = nn.LayerNorm(self.hidden_size, eps=eps)\n",
    "        self.ffn_dropout = nn.Dropout(dropout_rate)\n",
    "\n",
    "    def forward(\n",
    "            self,\n",
    "            hidden_states,\n",
    "            attn_mask=None,\n",
    "            encoder_outputs=None,\n",
    "            cross_attn_mask=None,\n",
    "    ):\n",
    "        \"\"\"\n",
    "        前向传播流程\n",
    "        \n",
    "        参数说明：\n",
    "        hidden_states: 输入隐藏状态（来自上一层或嵌入层）\n",
    "        attn_mask: 自注意力掩码（防止关注pad位置或未来信息）\n",
    "        encoder_outputs: 编码器输出（仅解码器需要）\n",
    "        cross_attn_mask: 交叉注意力掩码（通常用于解码器-编码器对齐）\n",
    "        \"\"\"\n",
    "        # ===== 阶段1：自注意力处理 =====\n",
    "        # 计算自注意力（Q=K=V=hidden_states）\n",
    "        self_atten_output = self.self_atten(\n",
    "            hidden_states, hidden_states, hidden_states, attn_mask\n",
    "        )\n",
    "        # 残差连接 + Dropout + 层归一化\n",
    "        # 残差设计：缓解梯度消失，保留原始信息\n",
    "        self_embeds = self.self_ln(\n",
    "            hidden_states + self.self_dropout(self_atten_output.hidden_states)\n",
    "        )  #多头注意力进行dropout，然后和原始输入进行残差连接，然后进行层归一化\n",
    "\n",
    "        # ===== 阶段2：交叉注意力处理（如果存在）=====\n",
    "        if self.cross_atten is not None:\n",
    "            assert encoder_outputs is not None\n",
    "            # 计算交叉注意力（Q=self_embeds, K=V=encoder_outputs）\n",
    "            cross_atten_output = self.cross_atten(\n",
    "                self_embeds, encoder_outputs, encoder_outputs, cross_attn_mask\n",
    "            )\n",
    "            # 残差连接 + Dropout + 层归一化\n",
    "            cross_embeds = self.cross_ln(\n",
    "                self_embeds + self.cross_dropout(cross_atten_output.hidden_states)\n",
    "            )\n",
    "\n",
    "        # ===== 阶段3：前馈神经网络处理 =====\n",
    "        # 如果有交叉注意力，则使用交叉注意力的输出作为FFN的输入；否则，使用self_embeds作为FFN的输入\n",
    "        embeds = cross_embeds if self.cross_atten is not None else self_embeds\n",
    "\n",
    "        ffn_output = self.ffn(embeds)  # 非线性变换\n",
    "        # 残差连接 + Dropout + 层归一化\n",
    "        embeds = self.ffn_ln(embeds + self.ffn_dropout(ffn_output))\n",
    "\n",
    "        return TransformerBlockOutput(\n",
    "            hidden_states=embeds,\n",
    "            self_attn_scores=self_atten_output.attn_scores,\n",
    "            cross_attn_scores=cross_atten_output.attn_scores\n",
    "            if self.cross_atten is not None\n",
    "            else None,\n",
    "        )"
   ],
   "id": "4afcbd9da9e207f2",
   "outputs": [],
   "execution_count": 365
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Encoder\n",
   "id": "3478d267cbaf5ab0"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.828071Z",
     "start_time": "2025-03-24T10:58:40.822182Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from typing import List  # 导入类型标注模块\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class TransformerEncoderOutput:\n",
    "    \"\"\"\n",
    "    编码器前向传播输出结果容器\n",
    "    \n",
    "    属性说明：\n",
    "    last_hidden_states: Tensor\n",
    "        经过所有编码器层处理后的最终隐藏状态（最后一层的输出）\n",
    "        [batch_size, seq_len, hidden_dim] 形状的张量\n",
    "        包含输入序列经过多层非线性变换后的高层语义表示\n",
    "    attn_scores: List[Tensor]\n",
    "        各编码器层的自注意力分数列表（每个元素对应一个层）\n",
    "        每个张量形状为 [batch_size, num_heads, seq_len, seq_len]\n",
    "        可用于可视化注意力模式或进行注意力分析\n",
    "    \"\"\"\n",
    "    last_hidden_states: Tensor  # 类型标注：张量\n",
    "    attn_scores: List[Tensor]  # 类型标注：张量列表\n",
    "\n",
    "\n",
    "class TransformerEncoder(nn.Module):\n",
    "    \"\"\"Transformer编码器模块，由多个堆叠的Transformer块组成\"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        \"\"\"\n",
    "        参数说明：\n",
    "        config: dict\n",
    "            模型配置字典，必须包含：\n",
    "            - num_encoder_layers: 编码器层数（如6层）\n",
    "            - 其他TransformerBlock需要的参数（通过config传递给各层）\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        # ===== 结构初始化 =====\n",
    "        # 从配置获取编码器层数（如BERT-base为12层）\n",
    "        self.num_layers = config[\"num_encoder_layers\"]\n",
    "\n",
    "        # 创建层堆叠：使用ModuleList确保参数正确注册\n",
    "        # layers,仅仅是一个模块的列表，它本身没有定义前向传递（forward pass）过程。需要在 forward 方法中明确地定义如何使用这些模块。\n",
    "        self.layers = nn.ModuleList(\n",
    "            [TransformerBlock(config) for _ in range(self.num_layers)]  # 创建N个独立Transformer块\n",
    "        )\n",
    "        # 使用ModuleList而不是普通Python列表的原因：\n",
    "        # 1. 确保PyTorch能正确识别子模块\n",
    "        # 2. 参数可以被优化器正确追踪\n",
    "        # 3. 支持模型保存/加载等操作\n",
    "\n",
    "    def forward(\n",
    "            self, encoder_inputs_embeds, attn_mask=None\n",
    "    ) -> TransformerEncoderOutput:\n",
    "        \"\"\"\n",
    "        编码器前向传播过程\n",
    "        \n",
    "        参数说明：\n",
    "        encoder_inputs_embeds: Tensor\n",
    "            经过嵌入层和位置编码的输入张量\n",
    "            形状：[batch_size, seq_len, hidden_dim]\n",
    "        attn_mask: Optional[Tensor]\n",
    "            自注意力掩码张量（可选），用于屏蔽填充token等\n",
    "            形状：[batch_size, 1, 1, seq_len] 或 [batch_size, seq_len]\n",
    "        \n",
    "        返回：\n",
    "            TransformerEncoderOutput 实例，包含最终隐藏状态和各层注意力分数\n",
    "        \"\"\"\n",
    "        attn_scores = []  # 初始化空列表用于收集各层注意力分数\n",
    "        embeds = encoder_inputs_embeds  # 初始输入（嵌入+位置编码）\n",
    "\n",
    "        # ===== 逐层处理 =====\n",
    "        for layer in self.layers:\n",
    "            # 通过当前层进行前向传播\n",
    "            block_outputs = layer(embeds, attn_mask=attn_mask)\n",
    "\n",
    "            # 更新隐藏状态：将当前层输出作为下一层输入\n",
    "            embeds = block_outputs.hidden_states  # 形状保持不变 [batch, seq_len, hidden]\n",
    "\n",
    "            # 收集当前层的自注意力分数（用于后续分析/可视化）\n",
    "            attn_scores.append(block_outputs.self_attn_scores)  # 添加 [batch, heads, seq, seq]\n",
    "\n",
    "        # ===== 返回最终结果 =====\n",
    "        return TransformerEncoderOutput(\n",
    "            last_hidden_states=embeds,  # 最终层输出\n",
    "            attn_scores=attn_scores  # 各层注意力分数列表\n",
    "        )"
   ],
   "id": "2c2454726a3a7034",
   "outputs": [],
   "execution_count": 366
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Decoder",
   "id": "e20f268997b8350d"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.833623Z",
     "start_time": "2025-03-24T10:58:40.828071Z"
    }
   },
   "cell_type": "code",
   "source": [
    "@dataclass\n",
    "class TransformerDecoderOutput:\n",
    "    \"\"\"\n",
    "    解码器前向传播输出结果容器\n",
    "    \n",
    "    属性说明：\n",
    "    last_hidden_states: Tensor\n",
    "        经过所有解码器层处理后的最终隐藏状态（形状：[batch_size, tgt_seq_len, hidden_dim]）\n",
    "        用于后续的词表映射（如生成目标语言的单词概率分布）\n",
    "    self_attn_scores: Tensor\n",
    "        各解码器层的自注意力分数列表（每个元素对应一个层的注意力矩阵）\n",
    "        每个张量形状：[batch_size, num_heads, tgt_seq_len, tgt_seq_len]\n",
    "        用于分析目标序列内部的关联模式（如关注前文哪些词）\n",
    "    cross_attn_scores: Optional[Tensor] = None\n",
    "        各解码器层的交叉注意力分数列表（每个元素对应一个层的注意力矩阵）\n",
    "        每个张量形状：[batch_size, num_heads, tgt_seq_len, src_seq_len]\n",
    "        反映目标序列与源序列的跨语言/跨模态对齐关系\n",
    "    \"\"\"\n",
    "    last_hidden_states: Tensor\n",
    "    self_attn_scores: List[Tensor]\n",
    "    cross_attn_scores: List[Tensor]\n",
    "\n",
    "\n",
    "class TransformerDecoder(nn.Module):\n",
    "    \"\"\"Transformer解码器模块，由多个堆叠的解码器块组成（每个块包含自注意力、交叉注意力和FFN）\"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        \"\"\"\n",
    "        参数说明：\n",
    "        config: dict\n",
    "            模型配置字典，必须包含：\n",
    "            - num_decoder_layers: 解码器层数（如6层）\n",
    "            - 其他TransformerBlock需要的参数（通过config传递给各层）\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        # ===== 结构初始化 =====\n",
    "        # 从配置获取解码器层数（如NMT标准模型使用6层）\n",
    "        self.num_layers = config[\"num_decoder_layers\"]\n",
    "\n",
    "        # 创建解码器层堆叠：每个层都启用交叉注意力机制\n",
    "        self.layers = nn.ModuleList(\n",
    "            [\n",
    "                TransformerBlock(config, add_cross_attention=True)\n",
    "                for _ in range(self.num_layers)\n",
    "            ]\n",
    "        )\n",
    "        # ModuleList的作用：\n",
    "        # 1. 确保所有子模块参数被正确注册到PyTorch中\n",
    "        # 2. 支持层间参数共享（若需要）\n",
    "        # 3. 允许通过索引访问特定层（如可视化中间层行为）\n",
    "\n",
    "    def forward(\n",
    "            self,\n",
    "            decoder_inputs_embeds,\n",
    "            encoder_outputs,\n",
    "            attn_mask=None,\n",
    "            cross_attn_mask=None,\n",
    "    ) -> TransformerDecoderOutput:\n",
    "        \"\"\"\n",
    "        解码器前向传播过程\n",
    "        \n",
    "        参数说明：\n",
    "        decoder_inputs_embeds: Tensor\n",
    "            目标序列的嵌入表示（嵌入层 + 位置编码）\n",
    "            形状：[batch_size, tgt_seq_len, hidden_dim]\n",
    "        encoder_outputs: Tensor\n",
    "            编码器的最终输出（源序列的高层表示）\n",
    "            形状：[batch_size, src_seq_len, hidden_dim]\n",
    "        attn_mask: Optional[Tensor]\n",
    "            自注意力掩码（防止查看未来token + 屏蔽padding）\n",
    "            形状：[batch_size, tgt_seq_len] 或 [batch_size, 1, tgt_seq_len, tgt_seq_len]\n",
    "        cross_attn_mask: Optional[Tensor]\n",
    "            交叉注意力掩码（通常用于屏蔽源序列的padding位置）\n",
    "            形状：[batch_size, src_seq_len] 或 [batch_size, 1, 1, src_seq_len]\n",
    "        \"\"\"\n",
    "        # 初始化注意力分数收集容器\n",
    "        self_attn_scores = []  # 存储所有层的自注意力分数\n",
    "        cross_attn_scores = []  # 存储所有层的交叉注意力分数\n",
    "\n",
    "        # 初始输入：目标序列嵌入（包含词嵌入和位置编码）\n",
    "        embeds = decoder_inputs_embeds\n",
    "\n",
    "        # ===== 逐层处理 =====\n",
    "        for layer in self.layers:\n",
    "            # 通过当前解码器层进行前向传播\n",
    "            block_outputs = layer(\n",
    "                embeds,\n",
    "                attn_mask=attn_mask,  # 自注意力掩码（如因果掩码）\n",
    "                encoder_outputs=encoder_outputs,  # 编码器的输出作为交叉注意力的K/V\n",
    "                cross_attn_mask=cross_attn_mask  # 交叉注意力掩码（如源序列padding掩码）\n",
    "            )\n",
    "\n",
    "            # 更新隐藏状态：将当前层输出作为下一层输入\n",
    "            embeds = block_outputs.hidden_states  # 形状保持不变 [batch, tgt_seq, hidden]\n",
    "\n",
    "            # 收集当前层的注意力分数\n",
    "            self_attn_scores.append(block_outputs.self_attn_scores)  # 自注意力分数\n",
    "            cross_attn_scores.append(block_outputs.cross_attn_scores)  # 交叉注意力分数\n",
    "\n",
    "        # ===== 返回最终结果 =====\n",
    "        return TransformerDecoderOutput(\n",
    "            last_hidden_states=embeds,  # 最终层输出（用于生成预测）\n",
    "            self_attn_scores=self_attn_scores,  # 所有层的自注意力分数\n",
    "            cross_attn_scores=cross_attn_scores  # 所有层的交叉注意力分数\n",
    "        )"
   ],
   "id": "fe1006487bbde678",
   "outputs": [],
   "execution_count": 367
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "## mask\n",
    "\n",
    "- mask实际上大类上只有两种\n",
    "    1. `padding_mask`：mask掉`pad_idx`，不计算损失\n",
    "    2. `attention_mask`：mask掉`pad_idx`，不计算注意力分数\n",
    "- Decoder的`attention_mask`和Encoder有一定的区别：\n",
    "    - Encoder可以同时看见序列所有信息，故只mask掉`pad_idx`\n",
    "    - Decoder只能看到在自身之前的序列的信息，故要额外mask掉自身之后的序列"
   ],
   "id": "5e3ee320a0c2b38a"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:40.963506Z",
     "start_time": "2025-03-24T10:58:40.833623Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def generate_square_subsequent_mask(sz: int) -> Tensor:\n",
    "    \"\"\"\n",
    "    生成一个正方形的掩码矩阵，掩码位置填完True，未掩码位置填完False。\n",
    "    \"\"\"\n",
    "    # torch.ones(sz, sz): 创建一个全为 1 的 sz × sz 的矩阵。\n",
    "    # torch.triu(...): 使用 triu 函数取得矩阵的上三角部分，将主对角线以下部分置零。\n",
    "    mask = (torch.triu(torch.ones(sz, sz)) == 0).transpose(-1, -2).bool()\n",
    "    # mask = torch.triu(torch.ones(sz, sz))\n",
    "    return mask\n",
    "\n",
    "\n",
    "plt.matshow(generate_square_subsequent_mask(16))\n",
    "plt.colorbar()\n",
    "plt.xlabel(\"keys\")\n",
    "plt.ylabel(\"querys\")\n",
    "plt.title(\"1 means mask while 0 means unmask\")\n",
    "plt.show()"
   ],
   "id": "c73124e6ffd16031",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 480x480 with 2 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbEAAAGZCAYAAAAHLw/qAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAPZxJREFUeJzt3Q2cTPX+B/DvLHbX89PmYVmRRB7X0wop5WHrulLqhoRLkSK0EluxSiwllyKipG6JbiFJJBHCXQ/pcvNYWE+LvdWuh2uXnfN/fX63mf/M7OzuzM6xc86Zz/u+zs2cnTnzm5kz8z2/3+/7+/1smqZpQkREZEJhwS4AERFRYTGIERGRaTGIERGRaTGIERGRaTGIERGRaTGIERGRaTGIERGRaTGIERGRaTGIERGRaTGIUdAsWrRIbDab7Ny5s8ies2PHjtK4ceMC73fs2DFVNpTRYeLEiWof0fXiOMfS09ODXRTTMHUQu3jxoiQlJck999wjlSpVyvWjQ2R0WVlZMnbsWImOjpaSJUtKmzZtZN26dcEuFpFpmDqI4Wrl5Zdflv3790uzZs2CXRyykBtvvFH++9//Sr9+/a7r8/z1r3+VGTNmSN++fWXWrFlSrFgx+dOf/iRbtmy5rs9LZBXFxcSqV68uZ86ckWrVqqkmqdatWwe7SGQRqNVHRkZe1+dISUmRJUuWyGuvvSbPPvus2te/f3/V3Pncc8/J1q1br+vzE1mBqWtiERERKoAFchVcpkwZSU1NlT//+c/q3zVq1JA5c+aov+/du1fuvvtuKV26tLoyX7x4ca5j/P777zJq1CiJiYlR5bn55ptl2rRpYrfb3e43ffp0adeunVSuXFk1G7Vs2VI+/fRTrz+ew4cPlxUrVqgfMxyzUaNGsmbNGrf7XbhwQT1v7dq11X2qVKkiXbp0kd27d/vU5n7o0CF59NFHpXz58nLDDTfI+PHjBQsanDhxQnr06CHlypVT7+3rr7/u9vjs7GyZMGGCKj8ei/emQ4cOsmHDhlzPhR9o3K9s2bLqeE2aNFG1jfz89ttvEhcXJzVr1pSDBw96vQ/ec9RY3njjDbdaeVhYmHp/XRdmePLJJ72eIz/99JPcddddUqpUKfWZv/rqqwX2ieXlww8/VK8TnyuatXv37q3ex4Lg88frGDJkiHMfAudjjz0m27ZtK/AYjv69f/3rX3LnnXeq14Lzz3Fefffdd6p5EuWqX7++fPPNN7mOcerUKRk0aJBUrVrVea4tXLiwUJ+54z3DuT5//nypW7euOiYuLnfs2OF237S0NBk4cKD6nHEfXJDivMMxCnrN2Lx9l/FdKExZAv0d+PXXX9VFCM5vPBbn+r333is//vhjrnK++eab6j3GZ1WxYkVp1aqV198VV8ePH1efKz7rs2fPilFt2rRJunfvrprG8d7jN6wgGzdulBYtWjh/OwvTHWTqIKaHnJwcdcIhCOGHDF8EBBG8mehrw0mGoIQfYlwlHz161PnYy5cvqx8P/Ijhb/hRbd++vSQmJkpCQoLb8+DHu3nz5qr5c8qUKVK8eHH5y1/+Il9++WWuMqEp6amnnlI/hijTlStX5MEHH5T//Oc/zvsMHTpU5s6dq/a/9dZb6kuEHys0rfqiV69eKtBOnTpV/dC98sorMnPmTBUI8QXGa8ZJhePi5HTIzMyUd955R/2Q4D4IiufPn5f4+HjZs2eP837o1+nTp4/6ouJ+eB485vvvv8+zTAhE+LHAFxU/wPjh9aZChQrqC+1aLrxn+OLgBwUBymHz5s3qB9czUOKzRRM0gnSDBg1Uv9RXX30l/po8ebL67OvVq6eaBXFhsX79ernjjjtUsM3PDz/8ILfccov60XOFIA6u72de8Frww4vPEOcKfgxw3ixdulT9F02TeO8vXbokDz30kLr4ccD7fNttt6nghnMe5yg+cwRRnAv+fuYO+FFG7fKJJ55Q5xUCSs+ePeXq1avO++C8Xb58uQpkOH9HjBihyoZAoidfyhLo78Avv/yifrDxOeAcGDNmjAp8+G04ffq0834LFixQr7Nhw4bq/X3ppZckNjZW/vnPf+ZZ/p9//lmdS3he/ODjYsOoLl26pL5TjuBfELyH3bp1UxeTOI/w3Xn88cdl7dq1/j2xZhE7duzA5bf23nvv+fyYAQMGqMdMmTLFue+3337TSpYsqdlsNm3JkiXO/QcOHFD3TUpKcu6bNGmSVrp0ae3QoUNuxx03bpxWrFgxLTU11bnv8uXLbvfJzs7WGjdurN19991u+/Ec4eHh2pEjR5z7fvzxR7X/zTffdO4rX768NmzYMM1fKD+ONWTIEOe+a9euaTVr1lSveerUqbneC7xPrvfNyspyOybuV7VqVW3QoEHOfSNHjtTKlSun7p8XfFYoCz67M2fOaI0aNdJuuukm7dixYwW+Drx2PKdDQkKCdscdd2hVqlTR5s6dq/b95z//Ua9p1qxZzvvdeeed6jk/+OAD5z68nmrVqmkPPvigc9/Ro0dznU+O984B5cTnPHnyZLey7d27VytevHiu/Z7wej0/f/j3v/+tnmfevHn5Pt7xWhYvXpzrPA0LC9O2b9/u3L927dpcr+exxx7TqlevrqWnp7sdt3fv3ur8cpyzvn7mjvescuXK2q+//urc//nnn6v9X3zxhfOxuP3aa6/l+/ryes3YPOEcvfHGG/0uix6/A1euXNFycnLcyoPnj4iI0F5++WXnvh49eqjPPD+Oc+z8+fPa/v37tejoaK1169Zur8EMRERbvnx5vvd57rnncr0fvXr10uLj4/16rpCviQGiv+tVPmoAaDp4+OGHnfuxD3/DVZfDP/7xD3WVj9oGahGOrXPnzurKzrWmgFqS69VzRkaGeqy35j88Hs0fDk2bNlVX667PjbLgCs71Sq+wrxlNWrjSxLmHq3DP98L1eXHf8PBw9W/U5FDzuXbtmnq862vBY3Fl5kum3cmTJ9VVK66O8Z6hyaYgeO9Qk3A0OaLGhStW7Me/HbUzvCbPmhiafNCU6oDXg9qP6+v0xbJly9R7gPPE9fNH8yVqZt6aWF0hcQQ1J0+Ovjj8vSB4LahxeZ6nt956q6qdOTj+7XiNeF8+++wz1fyDf7uWHzUsnJ+Oz9PXz9y1lo/vhIPj/Xc8N74LOB5qFvguXE8FlUWP3wF8hmjKBnzv0WKCzwX39fxO4Fz3bM70Zt++feo7gRohasqur8Eqtm3bpn7rXOHcw/6QSezQA34w0CfkCu3+aKv3HBOE/a5fusOHD6v+CM/HO5w7d87571WrVqnmDFSbkVbt4G3cUa1atXLtw0ns+txo8hgwYIBq/kBfBZqN0Mxx0003+fS6PZ8Drw3vRVRUVK79rs2Y8P7776tmuAMHDrg1y9SpU8f5bzSHfvLJJ6qJBs2TXbt2VT8GaJrxhAxANK+iKdTXPk7HjxECFj4rNM3h/cVngX4Qx98Q/D0zV719tnh/8Vn6A58/AgACljclSpTI9/H4MXc9FxzQfOz4e0HyOk9xXnjuA8c5hOZANHeivwhbQeevL595XueW4wfY8dz40UfT3OjRo1XzGJo00RSH8zeQPm5vCiqLHr8DCOxoikWzKJrIEMgc0EfrgCZrBCRcMKHZFt+JRx55RHVBeMLFBd4bNK0hIPrjypUrqh9TDzi/PV8/Pj9vF1/+Qr+oZ/MobqP5Ghdwvpz/EPJBDFeZ/ux3TRrAyYs+JGSSeYP+DseP6X333adqCjjR0YmNH7j33nvPa6euL8+NgIAfcvQrfP3116rdHz8MqB0gcBTE23P48rzo/0NH+P3336/a/pFQgsclJyer9nsH7EfAxpcQfU3Y8HrxQ4UfRFfoo/jggw/UDwGO4wt0HuMHFDU3XK2ijG3btlU/RCNHjlSd4XjfkUzjuEr253X6Ap8/vuB4bd6OWdCPD84DJFZ4Qsat4zVer/PXkXiEGikuhrxBC4A/n7mvzw3o/8APNfqScI4gsQjH+/bbb1XfcV7wfnv7nFwDh79lye9+vjwefdwoPxJkJk2apJJ7cM7hNbomeKF2jJYDXNAiUQs1YfweIGkG/WOu0GeI78lHH32k+vP8CWB1biwjaee8vx/+wjmM8biuMDYX/aJGEfJBLBBo8sMH7Fkl9oSTFVd6+LK6XsHgRz0Q+BFEjQcbrpqR5YNEA1+CWGEh8w21PQRL1ys0nNie0GSEHyps+DKjnG+//bb6wuNK1OHpp59Wt/FlxlXuuHHjfCoLgjiCGIIZOsjR+Y1aF46BHwk05Xj+OOj9+ePHDM/vuGDxB8qMJkdcebomdzg6+vH36wXBHu8XfvwLOn/9+cz9ff9QG8OGWi1eL2p7CJp5QU3KW1MgLlqCBe8PkhPeffddt/2o6Xq2bKB5Ek2c2FBbwgUcvrNIBnMd0oGLUrRO4DuDzwk1Nl9kZ2erAHZ0141SrmxgvUWZF+xSp+VxlSXren7qUQsD1Lo9sy1xG8/lay0M2CcWANSG0H7rLZsGJzD6DRxXc/jyu14tIkvKlxRUb3Ac9Fm4wtUxrty9NU/pyXFl6nolih9dz3ZszyZIXJk6ruy9lRGBDZmQ+DIj69LXIIb3EZl4juZFPA9qX8gSQ7OXZ3+YnvADhPcDgdLzyh63Pd8DT8gWxGfp2pyH9wYXN+jD8mwS1BPKjat9XGCh/8UTmhtd7+vLZ+4rZPU6mkxdAxp+rAs6f3E/NGm6lg+p7PllvV5veH88P3/0l3vWsj3PB1zkIVMRj/XMlsTvBc4LnCOoKa9cudKvMpUuo88GCCqum15BDC0nyOR1hT507A+pmtjs2bNVwHAkOHzxxReq89Rxhe/oC7ge0LSCkwvt+WhuQd8UkhmQXourM/zA4koMaaT4UUV/EK6oUGtCGipqH/72wwBSkdFWjxMcNQ9U+dHWjg5jz3FdesNrxRX5Aw88oF4X+gDmzZunvoyuzQ7oJEcCAFLmUVZcKWOMDK620aziDa4+EZyHDRumftBcky+8cQQoNNGgSccBzbZo4nOMC7pe8IOKfjgEXnzWaG5DufGeoJkX478cg5i9QaDCMAs8HucEzgc0IeFYnlf11wNS71ETRDkGDx6sPkN8ZqjB4nzCv/35zH2FMYqdOnVSF4E4BmoceL9wFe6apOINmuzwXUICAJKQ8L6hLBh7hRptMOD9wdAZDBfABRS+/2gG9OyfRh8Yah/oA0PfD/qA8fuF9xTnjSdckKFWivMK79Xq1avV98moLl68KEeOHHHexnmCLgU0r6JvEuc5Aju6DhzDhPD60R2DzxVNyehH9zbsKF+aySGtFi/D24Y01/wgtRYp8p6QwustFRbP1a1bN7d9Fy5c0BITE7Wbb75ZpcZHRUVp7dq106ZPn67S6B3effddrV69eirttkGDBirV2TNlG3DbW+o8ntuR6o505zFjxmjNmjXTypYtq14D/v3WW28V+H65pvAW5r2w2+0qFRnlwWtp3ry5tmrVqlwpzp9++qnWtWtXlfKO96VWrVraE088oVLpvaXYOyBVuU+fPipFfcWKFQW+Hhwfxzh79qxz35YtW9S+Dh06FPh6CkrRzi/F3uGzzz7Tbr/9dvX+YcPni8/w4MGDBZb/v//9r/bss8+qFH+8n0inXrNmTYGP8/c8zevcwvuGfTExMVqJEiVUOTp16qTNnz/f78/c8Z55S513TUtHSj+eE+8T3i+k87dp00b75JNPfHrdH374oRqKgfMqNjZWDR8obFn0+B1Aiv3o0aPVcAWk5bdv317btm1bruEAb7/9thoGgrR/vI9169ZV3+OMjIx8v58Y6oDjlClTxm3YhDcZGRnq8WkHa2mXT9cOaMMxcCzX8uVnw4YNXn+HHb9b+K/n8Ag8Bp8hPkt8pv4MkXKw4f/0jshERFT0MjMzVevT6YM1dekTi65/UrWOeA7INxL2iRERkWmZvk+MiIjc5Wia2gIR6OOLCoMYEZHF2EVTW6DHMAM2JxIRkWmxJkZEZDF20SQnRGpiDGJERBZjZ3MiERGR8bEmRkRkMTkhlJ3ImpiImgIKM6FjAk5MwZOSkhK0smAmb0yVhGloMB8ippxxrJllFJiuCHO7YZbuYMIUNpiaCstdYMJQLA+/c+fOoJYJcyFiHkhMCowyYWoqzGxelHMKFLRMPMqCyZYxgTTKiAmAMQFvsMqEeQOxTAk+P0yQi/tgtYPCrpWnR5k8YYok3Md1xWsjs+u0mUHIBzFMHpuQkKBm5MaccZiLEPOyua6lVJS+++47NXfg9u3b1WSY+IJjzjXMyWgEmJ8RM9E7JvMNFqznhDnosKQN5kn86aef1LyRwV48EMvhYAJjzAmHufFwG2u/Yd5IoywTj/K88cYbas5BTOSLwIFz3nNS3qIqEyYExncPwR//xTyNuHDD8kXBfJ8cMK8jvo++LI1jFDl/JHYEupmCFuLi4uLc5pPD3H1YEjw5OVkzgnPnzqn5x7777rtgF0XNE4n5H9etW6fmQBs5cmTQyjJ27Fg1X6HRYE69QYMGue3r2bOn1rdvX0MsE495EDE/ouucgr///ruay+/jjz8OSpm8SUlJUfc7fvx4UMt08uRJrUaNGtq+ffvUnIl/+9vfNCPL+GPuxH/vr6KlnqwW0IZj+DN3YrCEdE0Ma+/s2rXLbT0lzByN24VdZkJvjiVXMBN0sKGGiBm3C1p/qihg9YBWrVqpWeDR7IqFFBcsWBDsYqlZzLG8BGZqdywTsmXLluu6xps/MLM4VtR1/Qwx1x6a0Y1yzjvOezTfVahQIWhlwBp4WHUcq1VglnwzydH02cwgpBM70tPTVR+GtyWysWZRsOFLhH4nNJs1btw4qGVZsmSJaupBc6IRYGFENNuhKfj5559X5RoxYoRaoymvlYqLAhb0xCSsDRo0UOtM4fzCood9+/YVI0AAA2/nvONvwYZmTfSR9enTJ6gTz6IpGMvE4LwyG7sOfVpm6RML6SBmdKj5YMFCXMkHE1Z2HTlypOqjc119NtgBHjUxxzpiqInhvUI/TzCDGNZDwlpSixcvVlfvWE8JFyLoTwlmucwCfcBYOwstfL4ujno9oIVm1qxZ6sLNdTVrMp6Qbk7EgpW4Wva2RDYWrwum4cOHy6pVq9SihVhUMpjwhUaiS4sWLdSVKTYkoCA5AP92XbG6qCCzDgsqusJim6mpqRJMaHpCbQyLOyLbDs1RzzzzjMo6NQLHeW3Ec94RwLCAKi6YglkL27x5szrnsZij45xHuUaPHq0ymY3OLjbJCXDDMcwgpIMYmp6wGrPrEtm4wsdtf5fI1guuQBHAkBGFlU6Rqh1sWIUXq9WiVuHYUAtCExn+7Vi+viihidVz6AH6oW688UYJJmTaoV/VFd4fnFdGgPMJwcr1nEfzJ7IUg3XOuwYwpPpjVWkMmwgmXHxg1XXXcx61aVykrF27VozOrumzmUHINyeiTwXNPPhRjouLU+NAkHqLpcaD1YSIpqjPP/9cjRVz9FOg8x1jeoIB5fDsk0NaNn5ogtVXh9oNkijQnIgfP4ztmz9/vtqCCeOO0AeGK3g0J/7www8yY8YMtfy6UZaJR/PmK6+8IvXq1VNBDant+IHGmMRglAm16oceekg13aH1ATV7x3mPv+NiMxjvk2cgxXAOXADUr1//upSHCinY6ZFG8Oabb2q1atVSS2Qj5b6gJcCvJ2/Le2MrzLLd11OwU+zhiy++0Bo3bqzSw7HU/fz587Vgy8zMVO8LzqfIyEi15PoLL7ygZWVlFVkZClomHmn248eP16pWrareu06dOmkHDx4MWpmOHj2a53mPxwWjTN6YKcX+n/+upv07NTqgDccwQ4q9Df9X2ABIRETGkZmZqVpttv67upQpG1hv0cULdmnX6Iwa7hDM/smChHSfGBERmVvI94kREVmNXbOpLdBjmAGDGBGRxeT8kSYf6DHMgM2JRERkWqyJERFZTI6EqS2wY5gDgxgRkcVoOvSJ4RhmwCBGRGQxOewTCy1ZWVkyceJE9V+jMGKZjFoulsk3LJP5y0W5cbCzywBBIw3qM2KZjFoulsk3LJP5y+Vrub/6Vx0pHeBg50sX7HJv06OGfw/YnEhEZDF2NQt9YEHMrmbhMj42JxIRkWlZviaGJTBOnz6tZmLPa3E7VMFd/2sERiyTUcvFMvmGZTJuudCrc+HCBbWagOdSPoWRE0KJHZbvEzt58qTExMQEuxhERD6toh7IIriZf/SJLf+xnpQuG9g6f5cu5MgDzQ6zTyzYUAOD2+VPUlxKBHy85Yf26lAqIqL/l3nRLje2OOb8vSLfWT6IOZoQEcCK2wIPYuUCzPghIspLXl0ehUvssAV8DDOwfBAjIgo1dh2mnWJ2IhER0XVmiiA2Z84cqV27tkRGRkqbNm0kJSUl2EUiIjKsHC1Ml80MDF/KpUuXSkJCgiQlJcnu3bulWbNmEh8fL+fOnQt20YiIDNucaNdhMwPDl3LGjBkyePBgGThwoDRs2FDmzZsnpUqVkoULFwa7aEREFGSGTuzIzs6WXbt2SWJionMfBgJ27txZtm3b5vUxmLDTddJOow2iJCK63nI0m9oCPYYZGLomlp6eLjk5OVK1alW3/bidlpbm9THJyclqsJ9j40BnIgrVRTFzAtzMwByl9ANqbRhh7tgwAp6IKJTYtTBdNjMwdHNiVFSUFCtWTM6ePeu2H7erVavm9TERERFqIyIi6zN0qA0PD5eWLVvK+vXr3Sb0xe22bdsGtWxEREaVE0LNiYauiQHS6wcMGCCtWrWSuLg4mTlzply6dEllKxIRUW52HRIzcAwzMHwQ69Wrl5w/f14mTJigkjliY2NlzZo1uZI9iIgo9Bg+iMHw4cPVRkREBbPrMFjZLIOdTRHEiIjIdzk6TBvFaaeIiIiuM9bE/BQfHavLcdae3qPLcYiIPHE9MSIiMq0cNicSEREZH2tiREQWk6PDYGUOdiYioqCwaza1BXoMMzBHqCUiIvKCNTEiIoux69CcyMHOREQUFHYdllLhUixERBQUOWJTW6DHMANzhFoiIiIvWBMjIrIYO5sTiYjIrHJ0aA7EMczAHKGWiIjIC9bEiIgsxs7mRCIiMqscTgBMRETknzlz5kjt2rUlMjJS2rRpIykpKfnef+bMmVK/fn0pWbKkxMTEyDPPPCNXrlzx6zkZxIiILEb7Yz2xQDYcwx9Lly6VhIQESUpKkt27d0uzZs0kPj5ezp075/X+ixcvlnHjxqn779+/X9599111jOeff96v52UQIyKyaHNiToCbP2bMmCGDBw+WgQMHSsOGDWXevHlSqlQpWbhwodf7b926Vdq3by+PPPKIqr117dpV+vTpU2DtzRP7xEy+QjRwlWgiul4yMzPdbkdERKjNVXZ2tuzatUsSExOd+8LCwqRz586ybds2r8dt166dfPjhhypoxcXFyS+//CKrV6+Wfv36+VU+BjEiIoux67gUC/qqXKH5b+LEiW770tPTJScnR6pWreq2H7cPHDjg9fiogeFxt99+u2iaJteuXZOhQ4f63ZzIIEZEZDE5Oi6KeeLECSlXrpxzv2ctrLA2btwoU6ZMkbfeekslgRw5ckRGjhwpkyZNkvHjx/t8HAYxIiLKEwKYaxDzJioqSooVKyZnz55124/b1apV8/oYBCo0HT7++OPqdpMmTeTSpUsyZMgQeeGFF1RzpC+Y2EFEZNHmRHuAm6/Cw8OlZcuWsn79+v8vg92ubrdt29brYy5fvpwrUCEQApoXfcWaGBGRxdglLOBFLf19PNLrBwwYIK1atVKJGhgDhpoVshWhf//+UqNGDUlOTla3u3fvrjIamzdv7mxORO0M+x3BzPRBDC922bJlqmMQg+GQzTJt2jQ1OI6IiLzL0WxqC4S/j+/Vq5ecP39eJkyYIGlpaRIbGytr1qxxJnukpqa61bxefPFFsdls6r+nTp2SG264QQWwyZMn+/W8Ns2felsRu+eee6R3797SunVrlbmCrJV9+/bJTz/9JKVLl/Y5PbR8+fLSUXpIcVsJsSKm2BOZW+YFu1S85RfJyMgosP/Jl9+7Jzf3lIgygf3eZV28KnM7LAu4TNeboWtiiOKuFi1aJFWqVFHjEe64446glYuIKFRS7I3O0EHME64IoFKlSnneJysrS215DdQjIrI6TYdZ7HEMMzBHKf/IdBk1apSapqRx48b59qOhOu3YPAfqERGRdZgmiA0bNkz1hy1ZsiTf+2HaE9TYHBsG6hERhZIcsemymYEpmhOHDx8uq1atkk2bNknNmjXzva+3eb2IiEKJXQu8TwvHMANDBzEkTj799NOyfPlyNUVJnTp1gl0kIiIykOJGb0LEmjOff/65lC1bVo09APR1YdwYERHlZtchsSPQxxcVQ5dy7ty5ql+rY8eOUr16deeGhdOIiMi7QBfEdGxmYOiamIHHYRMRkQEYOogREZE5pp0KFgYxIiKLsYdQnxiDmAXER8fqchzOwUhEZsMgRkRkMXYkZgQ6ToyJHUREFAyaDtmFOIYZMIgREVmMPYRmsTdHzx0REZEXrIkREVmMndmJRERkVnY2JxIRERkfa2JERBZj1yE7kSn2REQUFHY2JxIRERkfa2JERBZjD6GaGIMYEZHF2EMoiLE5kYiITIs1MSIii7GHUE2MQYyIyGI0HVLkcQwzYBAjIrIYewjVxNgnRkREpsWaGOm+QjRwlWii4LGHUE2MQYyIyGLsIRTE2JxIRESmxZoYEZHF2EOoJsYgRkRkMZpmU1ugxzADNicSEZFpmSqITZ06VWw2m4waNSrYRSEiMvx6YvYANzMwTXPijh075O2335amTZsGuyhERIZmD6E+MVPUxC5evCh9+/aVBQsWSMWKFYNdHCIiMghTBLFhw4ZJt27dpHPnzgXeNysrSzIzM902IqJQTOzQAtzMwPDNiUuWLJHdu3er5kRfJCcny0svvXTdy0VEZFR2Nicaw4kTJ2TkyJHy0UcfSWRkpE+PSUxMlIyMDOeGYxARkTUZuia2a9cuOXfunLRo0cK5LycnRzZt2iSzZ89WTYfFihVze0xERITaiIhClRZC48QMHcQ6deoke/fudds3cOBAadCggYwdOzZXACMiIlEBKNDmQAYxHZQtW1YaN27stq906dJSuXLlXPuJiOj/F7TUAlzV0iyLYhq6T4yIiMi0NTFvNm7cGOwiEBEZml1s6n+BHsMMTBfEiIgof1oIJXawOZGIiEyLNTG6LuKjY3U71trTe3Q7FlEosGs2sYXIYGcGMSIii9E0HbITTZKeyOZEIiIyLdbEiIgsRguhxA4GMSIii9FCKIixOZGIiEyLNTEiIouxMzuRiIjMSmN2IhERkfGxJkZEZMmamC3gY5gBgxgRkcVoIZSdyCBGRGTF9cQk8GOYAfvEiIjItFgTIyKyGI3NiUREZFpa6LQnsjmRiIh0MWfOHKldu7ZERkZKmzZtJCUlJd/7//777zJs2DCpXr26REREyC233CKrV6/26zlZEyMishot8OZEHMMfS5culYSEBJk3b54KYDNnzpT4+Hg5ePCgVKlSJdf9s7OzpUuXLupvn376qdSoUUOOHz8uFSpU8Ot5GcSIiCxGC8KMHTNmzJDBgwfLwIED1W0Esy+//FIWLlwo48aNy3V/7P/1119l69atUqJECbUPtTh/sTmRiIjylJmZ6bZlZWV5rVXt2rVLOnfu7NwXFhambm/bts3rcVeuXClt27ZVzYlVq1aVxo0by5QpUyQnJ0f8wZoYGV58dKwux1l7eo8uxyEKpezEmJgYt/1JSUkyceJEt33p6ekq+CAYucLtAwcOeD3+L7/8It9++6307dtX9YMdOXJEnnrqKbl69ap6Dl8xiBERWY1m87tPy+sxROTEiRNSrlw5524kYOjBbrer/rD58+dLsWLFpGXLlnLq1Cl57bXXGMSIiEgfCGCuQcybqKgoFYjOnj3rth+3q1Wr5vUxyEhEXxge53DrrbdKWlqaap4MDw/3qXzsEyMismhihxbg5isEHNSk1q9f71bTwm30e3nTvn171YSI+zkcOnRIBTdfAxgwiBERWXWwsxbg5gek1y9YsEDef/992b9/vzz55JNy6dIlZ7Zi//79JTEx0Xl//B3ZiSNHjlTBC5mMSOxAooc/DB/E0Eb66KOPSuXKlaVkyZLSpEkT2blzZ7CLRURELnr16iXTp0+XCRMmSGxsrOzZs0fWrFnjTPZITU2VM2fOOO+PhJG1a9fKjh07pGnTpjJixAgV0Lyl45u2T+y3335TVc677rpLvvrqK7nhhhvk8OHDUrFixWAXjYjIsLQgzZ04fPhwtXmzcePGXPvQ1Lh9+3YJhKGD2LRp01S0fu+995z76tSpE9QyERGZgiYhwdDNiRgM16pVK/nLX/6iUjGbN2+u2lyJiChvjppYoJsZGDqIYTDc3LlzpV69eqrtFB2BaDdFx2FeMJrcc4Q5ERFZk6GbE5F6iZoYMlYANbF9+/apObkGDBjg9THJycny0ksvFXFJiYgMRONSLIaA8QINGzZ024fBcMhyyQtSODMyMpwbRpsTEYUWm06b8Rm6JobMREzj7wrjCW688cY8H4MpUfSaFoWIiIzN0DWxZ555RqVfojkRI7sXL16s5tnydzAcEVFI0Yp+sHOwGDqItW7dWpYvXy4ff/yxmqZ/0qRJaqE1zHpMRER5CKEgZujmRPjzn/+sNiIiIl1qYkiWOHnypPN2SkqKjBo1SjX1ERGRQZZi0QLcrBrEHnnkEdmwYYP6N6bN79KliwpkL7zwgrz88st6l5GIiAw8i73pmhMxVisuLk79+5NPPlH9Vd9//718/fXXMnToUDUBJJFVV4gGrhJNZOIghuWjHWns33zzjdx3333q3w0aNHCbpZiIiIJA42DnfDVq1EjNmrF582ZZt26d3HPPPWr/6dOn1ZIpREQURBr7xAqcXf7tt9+Wjh07Sp8+faRZs2bOCXsdzYxERESGbE5E8EpPT1eT67qu7TVkyBApVaqUnuUjIiI/2bT/bYEew7I1saSkJJVi77k4Ze3atdWSKUREFERa6Ax2LlQQ+/zzz6Vu3brSqVMnNRUUlj8hIiKD0Ngnlq89e/bIjh07VILHyJEjpVq1amqtL+wjIiIy/NyJWNvrjTfeUBmJ7777rmpexKzzTZs2lVmzZqllUIiIKAg0Nif6TNM0NW4sOztb/Rv9ZLNnz5aYmBhZunSpPqUkIiLfaQxiBdq1a5cMHz5cLVyJJVNQM9u/f7989913cvjwYZk8ebKMGDFC39ISEREFGsSaNGkit912mxw9elQ1JWJC4KlTp8rNN9/svA/Gj50/f74whyciokBooVMTK9Q4sYcfflgGDRokNWrUyPM+UVFRYrfbAykbEREVhqZDdqFVsxPR/7Vo0SI10JmIiMhUNbESJUrIlStXrk9piIgoYDbO2JG/YcOGqfkTr127pn+JiIgoMBr7xPKFQc3r169X64chyaN06dJuf1+2bJle5SMiItI3iFWoUEEefPDBwjyUiIgouEHsvffe068ERESkK5sOfVo2KwcxQH/Yxo0b5eeff5ZHHnlEypYtq6agKleunJQpU0bfUhIZTHx0rG7HWnt6j27HIgo1hQpix48fV6s5p6amqhnsu3TpooIYkj1wG6s+ExFRkGgcJ5YvzFzfqlUr+e2336RkyZLO/Q888IBK+CAioiDSmJ2Yr82bN8vWrVslPDw816KYp06d0qtsRERUGHoEIZMEsULVxDCdVE5OTq79WI4FzYpERESGDWJdu3aVmTNnOm/bbDa5ePGiJCUlyZ/+9Cc9y0dERIWcscMW4GbZIPb666/L999/Lw0bNlRTUCE70dGUiOQOvaC2N378eKlTp47qe6tbt65MmjRJrVtGRER5YJ9Y/mrWrCk//vijLFmyRP71r3+pWthjjz0mffv2dUv0CBQC4ty5c+X999+XRo0ayc6dO2XgwIFSvnx5rlVGRESFHydWvHhxefTRR+V6QvJIjx49pFu3buo2ansff/yxpKSkXNfnJSIyNS10EjsKFcQ++OCDfP/ev39/0UO7du1k/vz5cujQIbnllltU7W/Lli0yY8aMPB+DcWrYHLhkDBGFGlsIzWJfvLDjxDzXGLt8+bJKuS9VqpRuQWzcuHEqCDVo0ECKFSum+sgmT56smi3zkpycLC+99JIuz09ERBZM7MAgZ9cNfWIHDx6U22+/XTX36eWTTz6Rjz76SBYvXiy7d+9WfWPTp09X/81LYmKiZGRkOLcTJ07oVh4iIlPN2KEFuFm5T8xTvXr1ZOrUqaqf7MCBA7occ8yYMao21rt3b3Uby75gyivUtgYMGOD1MREREWojIgpZWuj0iRWqJpZfsgcmAdYLmijDwtyLiGZFDLYmIiIqVE1s5cqVbrcxbuvMmTMye/Zsad++vV5lk+7du6s+sFq1aqkU+x9++EEldQwaNEi35yAishobEzvyd//997vdxowdN9xwg9x9991qILRe3nzzTTXY+amnnpJz585JdHS0PPHEEzJhwgTdnoOIyHK00GlOLFQQK6rmPMzDiOmtXKe4IiKiAmg61KSsHMQSEhJ8vm9+Y7qIiIiKPIihbwop71jduX79+mofBiQj6aJFixZuzYxERFTENDYnFphwgaY+jNeqWLGi2ofxYpjXsEOHDjJ69Gi9y0lkWfHRsbocZ+3pPbochyxAC50gVuhZ7DFWyxHAAP9+5ZVXdE3sICIi0r0mhqmgzp8/n2s/9l24cKEwhyQiIp3YQijFvlA1sQceeEA1HS5btkyt5ozts88+U8ux9OzZU/9SEhER6VUTmzdvnjz77LNqMUxM/qsOVLy4CmKvvfZaYQ5JRERUNEEMM9W/9dZbKmD9/PPPah9WXS5dunRhDkdERHrSQiexI6AJgBG0mjZtql9piIgoYDb2iREREYXQUixERGQgmoQEBjEiIqvRQqdPjM2JRERkWqyJERFZjC2EEjsYxIiIrEYLneZEBjEiIouxhVBNjH1iRERkWgxiRERWbU7UAtz8NGfOHKldu7ZERkZKmzZtJCUlxafHLVmyRK0/ef/99/v9nAxiRERWoxV9EFu6dKkkJCRIUlKSWjS5WbNmEh8fL+fOncv3cceOHVNz8WItysJgECMiooDNmDFDBg8erFY4adiwoZooHvPsLly4MM/H5OTkSN++feWll16Sm266qVDPy8QOIovQa4Vo4CrR5mbTMbED60e6ioiIUJur7Oxs2bVrlyQmJjr3hYWFSefOnWXbtm15PsfLL78sVapUUSugbN68uVDlZE2MiMhqNP2aE2NiYqR8+fLOLTk5OdfTpaenq1pV1apV3fbjdlpamtcibtmyRd59911ZsGBBQC+VNTEiIsrTiRMnpFy5cs7bnrWwwrhw4YL069dPBbCoqKiAjsUgRkRkNZp+g50RwFyDmDcIRMWKFZOzZ8+67cftatWq5bo/1qFEQkf37t2d++x2u3OB5YMHD6o1Kn3B5kQiIov2idkC3HwVHh4uLVu2lPXr17sFJdxu27Ztrvs3aNBA9u7dK3v27HFu9913n9x1113q32jC9BVrYkREFDCk1w8YMEBatWolcXFxMnPmTLl06ZLKVoT+/ftLjRo1VJ8axpE1btzY7fEVKlRQ//Xcb+ia2KZNm1R1Mjo6Wg10W7FihdvfNU2TCRMmSPXq1aVkyZIq0+Xw4cNBKy8RkSloRT9OrFevXjJ9+nT1mx0bG6tqVGvWrHEme6SmpsqZM2d0f6lBrYkhSmNA3KBBg6Rnz565/v7qq6/KG2+8Ie+//77UqVNHxo8frwbP/fTTTyqSExGRceZOHD58uNq82bhxY76PXbRokfmC2L333qs2b1ALQ3X0xRdflB49eqh9H3zwgYrqqLH17t27iEtLRERGY9jEjqNHj6rxBWhCdMAYBczHld/guaysLDU4z3UjIgopWnDmTgwGwwYxxwA5fwbPAToNXQfm+ZPlQkRkCRqDmGlh2pOMjAznhoF6REShxKbTZgaGDWKOAXK+Dp5zHU3uGJznyyA9IiIyL8MGMWQjIli5Dp5D/9Y///lPr4PniIgo9JoTg5qdePHiRTly5IhbMgfGFlSqVElq1aolo0aNkldeeUXq1avnTLHHmLLCLJxGRBQqbEFKsQ+5ILZz5041zYjriG/AqG+MGXjuuefUWLIhQ4bI77//LrfffrsaPMcxYkREFPQg1rFjRzUeLC+YxQPrzWAjIqKinwDY6Dh3IhGRFWkSEgyb2EFERFQQ1sSIKJf46FhdjrP29B5djkP+sTGxg4iITEsLnT4xNicSEZFpsSZGRGQxNjYnEhGRaWlsTiQiIjI81sSIiCzGxuZEIiIyLS10mhMZxIiIrEYLnSDGPjEiIjIt1sSIiCzGxj4xIiIyLY3NiURERIbHmhgRkcXYNE1tgR7DDBjEiIisRmNzIhERkeGxJkZEZDE2ZicSEZFpaaHTnMggRkSGXyEauEo0ecMgRkRkMTY2JxIRkWlpodOcyOxEIiIyLdbEiIgsxsbmRCIiMi2NzYlFYtOmTdK9e3eJjo4Wm80mK1ascP7t6tWrMnbsWGnSpImULl1a3ad///5y+vTpYBaZiMhUtTFbITezCGoQu3TpkjRr1kzmzJmT62+XL1+W3bt3y/jx49V/ly1bJgcPHpT77rsvKGUlIiLjCWpz4r333qs2b8qXLy/r1q1z2zd79myJi4uT1NRUqVWrVhGVkojIZDTtf1ugxzABU/WJZWRkqGbHChUq5HmfrKwstTlkZmYWUemIiIzBFkKJHaZJsb9y5YrqI+vTp4+UK1cuz/slJyerWpxji4mJKdJyEhFR0TFFEEOSx8MPPyyapsncuXPzvW9iYqKqsTm2EydOFFk5iYgMlZ2oBbiZQHGzBLDjx4/Lt99+m28tDCIiItRGRBSqbPb/bYEewwyKmyGAHT58WDZs2CCVK1cOdpGIiMhAghrELl68KEeOHHHePnr0qOzZs0cqVaok1atXl4ceekil169atUpycnIkLS1N3Q9/Dw8PD2LJiYgMTAudwc5BDWI7d+6Uu+66y3k7ISFB/XfAgAEyceJEWblypbodG+u+nANqZR07dizi0hIRmYMthLITgxrEEIiQrJGX/P5GRERk6D4xIiIqBI2DnYmIyKRsbE4kIjKW+Gj3vvFArD29R7djUXAxiBERWY3G7EQiIjIpG5sTiYjItLTQSewwxdyJRERE3rAmRkRkMTY2JxIRkWlpoZPYweZEIiIyLdbEiIgsxsbmRCIiMi279r8t0GOYAJsTiYjItFgTIyKyGi10EjsYxIiILMamQ58WjmEGbE4kIiLTYk2MiMhqNE47RUREJk+xtwW4+WvOnDlSu3ZtiYyMlDZt2khKSkqe912wYIF06NBBKlasqLbOnTvne/+8MIgREVk1sUMLcPPD0qVLJSEhQZKSkmT37t3SrFkziY+Pl3Pnznm9/8aNG6VPnz6yYcMG2bZtm8TExEjXrl3l1KlTfj0vgxgREQVsxowZMnjwYBk4cKA0bNhQ5s2bJ6VKlZKFCxd6vf9HH30kTz31lMTGxkqDBg3knXfeEbvdLuvXr/freRnEiIgsxqZpumyQmZnptmVlZeV6vuzsbNm1a5dqEnQICwtTt1HL8sXly5fl6tWrUqlSJb9eKxM7iCjkxEfH6nKctaf3iCHZ/9gCPYaIauZzhebCiRMnuu1LT0+XnJwcqVq1qtt+3D5w4IBPTzd27FiJjo52C4S+YBAjIqI8nThxQsqVK+e8HRERIXqbOnWqLFmyRPWTISnEHwxiREQWY3NpDgzkGIAA5hrEvImKipJixYrJ2bNn3fbjdrVq1fJ97PTp01UQ++abb6Rp06Z+l5N9YkREVqMVbXZieHi4tGzZ0i0pw5Gk0bZt2zwf9+qrr8qkSZNkzZo10qpVq0K9VNbEiIgoYEivHzBggApGcXFxMnPmTLl06ZLKVoT+/ftLjRo1JDk5Wd2eNm2aTJgwQRYvXqzGlqWlpan9ZcqUUZspamKbNm2S7t27q848m80mK1asyPO+Q4cOVffBG0NERPlwzNgR6OaHXr16qaZBBCakze/Zs0fVsBzJHqmpqXLmzBnn/efOnauyGh966CGpXr26c8MxTFMTQ5TGgLhBgwZJz54987zf8uXLZfv27SrYERGRMRfFHD58uNq8QdKGq2PHjokeghrE7r33XrXlB6O3n376aVm7dq1069atyMpGRETGZ+g+MXQM9uvXT8aMGSONGjXy6TEYiOc6GA+D84iIQorGCYANAR1/xYsXlxEjRvj8GHQali9f3rl5DtQjIrI6m12fzQwMG8QwhcmsWbNk0aJFKqHDV4mJiZKRkeHcMFCPiIisybBBbPPmzWr241q1aqnaGLbjx4/L6NGjVTpmXjCa3DE4z5dBekRElqMVfXZisBi2Twx9YZ5zaGFaf+x3jDsgIiIvCrGUitdjmEBQg9jFixflyJEjzttHjx5VYwswizFqYJUrV3a7f4kSJdQUJvXr1w9CaYmIQm/aKaMLahDbuXOn3HXXXW4jvgGjvtEXRkREZNgg1rFjR9H8iPZ6DY4jIrI0LXRS7A3bJ0ZERIWk6bCemDlimHGzE4mIiArCmhgRUZBXiL6mXRWRX0QvNiZ2EBGRuVPstcCPYQJsTiQiItNiTYyIyGo0ZicSEZFZ2dGppcMxTIDNiUREZFqsiRERWYyN2YlERGRaWuj0ibE5kYiITIs1MSIiq9FCpybGIEZEZDUagxgREZmVnSn2REREhseaGBGRxdiYYk9ERKalhU6fGJsTiYjItFgTIyKyGruG9sDAj2ECDGJERFajhU5zouWDmPbHB3FNrppmkTciCi3q98nl94p8Z/kgduHCBfXfLbI62EUhIirw96p8+fI6HEnToSZljoBq+SAWHR0tJ06ckLJly4rN5n30X2ZmpsTExKj7lStXTozAiGUyarlYJt+wTMYtF2pgCGD4vdLpgMLmRIsICwuTmjVr+nRfnKxG+iIZtUxGLRfL5BuWyZjl0qcGFnosH8SIiEKOHbUoZicSEZEZafb/bYEewwQ42FlEIiIiJCkpSf3XKIxYJqOWi2XyDctk/nJRbjaNOZ1ERJaQmZmp+tY6xzwpxcMCC8DX7FnyzYm5kpGRYcj+Sgc2JxIRWY2dfWJERGRWWuik2LNPjIiITIs1MSIiq9F0qEmZoyLGmhiFro4dO8qoUaOCXQyi69ecqAW4mQCDGBERmRabE4mIrMaOgcp2HY5hfKyJEf3hyy+/VGNsPvroIzXx68MPPywVKlSQSpUqSY8ePeTYsWPqfps2bZISJUpIWlqa2+PRNNmhQwf17+PHj0v37t2lYsWKUrp0aWnUqJGsXs2VFKiIaGxOJAopixcvlj59+qgAhuAVHx+vVj7YvHmzfP/991KmTBm55557JDs7W+644w656aab5O9//7vz8VevXlWPHTRokLo9bNgwycrKUgFv7969Mm3aNHUMItIXmxMp5M2ZM0deeOEF+eKLL+TOO++UDz/8UOx2u7zzzjvO5Xvee+89VSvbuHGjdO3aVR577DG1b8yYMerveOyVK1dUAITU1FR58MEHpUmTJuo2gh5RkdFCZ5wYgxiFtE8//VTOnTunalutW7dW+3788Uc5cuSIqom5QpD6+eef1b//+te/yosvvijbt2+X2267TRYtWqQCGJoOYcSIEfLkk0/K119/LZ07d1YBrWnTpkF4hRSS7KEzYwebEymkNW/eXG644QZZuHChc2n4ixcvSsuWLWXPnj1u26FDh+SRRx5R96lSpYrq80Jt7OzZs/LVV185mxLh8ccfl19++UX69eunmhNbtWolb775ZtBeJ5FVsSZGIa1u3bry+uuvqzFjxYoVk9mzZ0uLFi1k6dKlKlDlN/EpAhX60bDoKo7Tvn17t79jZeChQ4eqLTExURYsWCBPP/10EbwqCnWaZldboMcwA9bEKOTdcsstsmHDBvnss89UhmHfvn0lKipKZSQisePo0aOqLwxNhCdPnnQ+DskfCHKvvPKKDBw40O2YOM7atWvVY3fv3q2Of+uttwbh1VFI0rT/NQcGsrFPjMg86tevL99++62zRoaswrFjx0rPnj3lwoULUqNGDenUqZNbzSwsLEz1jU2ZMkX69+/vdrycnByVoYigh8cgs/Fvf/tbEF4ZkbVxPTGiACBL8fz587Jy5cpgF4VIHOuJdSrfT4rbwgM61jUtW9Zn/J3riRFZEb7YSNjA+DIGMDIcu13EFmCflkn6xBjEiAoB/WUpKSkqaaNLly7BLg6RO9XAxnFiRJQHJHoQUfAxiBERWYxmt4tmC40UewYxIiKr0UKnOZHjxIiIyLRYEyMishq7hgFUIVETYxAjIrIaDQHIHhJBjM2JRERkWqyJERFZjGbXRAuwOdEskzmxJkZEZDWaXZ+tEAvM1q5dWyIjI6VNmzZqQoD8/OMf/5AGDRqo+2MB2dWrV/v9nAxiREQUMCxflJCQIElJSWrlhmbNmqmVHrDorDdbt25VSxlh/tEffvhB7r//frXt27fPr+flBMBERBabALij7QEpbisR0LGuaVdlo7bc5wmAUfPC6uhYkw/sdrtaUw9r6I0bNy7X/Xv16iWXLl2SVatWOfdhlfTY2FiZN2+ez+VkTYyIyGq0om1OzM7Oll27dknnzp3dlirC7W3btnl9DPa73h9Qc8vr/nlhYgcRkcVck6sBT9ihjvFH7c5VRESE2lylp6erNfSqVq3qth+3Dxw44PX4aWlpXu+P/f5gECMisojw8HCpVq2abEnzP0HCmzJlyqgmQVfo85o4caIYBYMYEZFFREZGytGjR1Xznh6QMmGz2dz2edbCICoqSq2IfvbsWbf9uI2g6g32+3P/vDCIERFZLJBFRkYWeQ2wZcuWsn79epVh6EjswO3hw4d7fUzbtm3V30eNGuXct27dOrXfHwxiREQUMKTXDxgwQFq1aiVxcXEyc+ZMlX04cOBA9ff+/ftLjRo1JDk5Wd0eOXKk3HnnnfL6669Lt27dZMmSJbJz506ZP3++X8/LIEZERAFDyvz58+dlwoQJKjkDqfJr1qxxJm+kpqaqjEWHdu3ayeLFi+XFF1+U559/XurVqycrVqyQxo0b+/W8HCdGRESmxXFiRERkWgxiRERkWgxiRERkWgxiRERkWgxiRERkWgxiRERkWgxiRERkWgxiRERkWgxiRERkWgxiRERkWgxiRERkWgxiREQkZvV/0HM9KFebrHoAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 368
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.302740Z",
     "start_time": "2025-03-24T10:58:40.963506Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#通过下面代码查看mask的效果\n",
    "inputs_words = [\"The quick brown fox jumps over the lazy dog .\", \"What does the fox say ?\"]\n",
    "\n",
    "inputs_ids, input_mask = tokenizer.encode([w.split() for w in inputs_words], return_mask=True)\n",
    "for i in range(len(inputs_words)):\n",
    "    decode_text = \\\n",
    "        tokenizer.decode(inputs_ids[i: i + 1].tolist(), remove_bos=False, remove_eos=False, remove_pad=False,\n",
    "                         split=True)[0]\n",
    "    print(decode_text)\n",
    "    self_attn_mask = input_mask[i].reshape(1, -1).repeat_interleave(inputs_ids.shape[-1], dim=0)\n",
    "    look_ahead_mask = generate_square_subsequent_mask(inputs_ids.shape[-1])\n",
    "\n",
    "    fig, axs = plt.subplots(1, 2, figsize=(10, 5))\n",
    "    axs[0].matshow(self_attn_mask)\n",
    "    axs[0].set_title(\"self_attn_mask\")\n",
    "    axs[0].set_yticks(range(len(decode_text)), decode_text, fontsize=6)\n",
    "    axs[0].set_ylabel(\"querys\")\n",
    "    axs[0].set_xticks(range(len(decode_text)), decode_text, fontsize=6)\n",
    "    axs[0].set_xlabel(\"keys\")\n",
    "    axs[1].matshow(look_ahead_mask)\n",
    "    axs[1].set_title(\"look_ahead_mask\")\n",
    "    axs[1].set_yticks(range(len(decode_text)), decode_text, fontsize=6)\n",
    "    axs[1].set_ylabel(\"querys\")\n",
    "    axs[1].set_xticks(range(len(decode_text)), decode_text, fontsize=6)\n",
    "    axs[1].set_xlabel(\"keys\")\n",
    "    plt.show()\n",
    "    print('-' * 50)"
   ],
   "id": "b7762b4c2e704123",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[BOS]', '[UNK]', 'quick', 'brown', '[UNK]', 'jumps', 'over', 'the', '[UNK]', 'dog', '.', '[EOS]']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1000x500 with 2 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1IAAAG1CAYAAADz+MUUAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAQQBJREFUeJzt3Qd8FNXa+PEnIRBpCYgIBKJ0pAoGBA0lXJrYQH0pggKCogheVEDJRUVRiYCgr1gQERRBynsFQa9Ik9ARUEJRUboJEQWBJKIEQub/eY7/3ZtGkknZ+vt+PpOdnZ2dMzObnWefM2fOBFiWZQkAAAAAIN8C8z8rAAAAAECRSAEAAACATSRSAAAAAGATiRQAAAAA2EQiBQAAAAA2kUgBAAAAgE0kUgAAAABgE4kUAAAAANhEIgUAAAAANpFIAZfxwQcfSEBAgBw9ejTT9ClTpkjt2rWlRIkS0rx5c7etn697/vnnzf4/deqUu1cFAPIdI4rKoEGDpFy5cuLq7dm5c6d4WhzwVsQx30ciBdiwatUqeeqppyQyMlLmzJkjEydOLNLlb9myxRx4z549m+01LevTTz8t0vIAAABQMCRSgA1fffWVBAYGyvvvvy8DBgyQW2+9tcgTqRdeeIFECgAAwMORSAE2/Pbbb1K6dGkpVaqUu1cFAAAAbkQiBZ+TkpIijz/+uNSsWVOCg4Pl6quvli5dusi3337rnOfrr7+WW265RUJDQ6VMmTLSoUMH2bx5c67L1XbO2pzv3LlzZlwHbVOeH3v27DHt3fXaqiuuuEKqVq0qgwcPlt9//905jzbpGzNmjBmvVauWswxtf6+PWu6HH37onK7Lc7xPnx88eNBMq1ChgtmuBx54QP78888Ctcn/+eef5fbbbzfj1atXl7feesu8vnfvXvnHP/4hZcuWlWuvvVY+/vjjTO8/ffq0jB49Wpo2bWreGxISIt27d5fdu3dnK2v69OnSuHFjs/8rVqwoLVu2zLa8rI4dOyZ169aVJk2ayK+//mpr2wCguL399tvmuKaxJywsTIYPH55jC4P/+7//k4iICFMxd9VVV8l9990nx48fz3P5cXFxUrlyZYmKipI//vgjX+ukx81HH31UGjRoYMqrVKmS9OrV67LXdqWmpsqTTz5pytFj/V133SUnT57MNt+KFSukXbt2Zp7y5cvLbbfdJt99953t2OewadMmadWqlZmvTp068u6770pBEMfgSkEuLQ1wgUceeUT+/e9/y4gRI6RRo0bmgK0H6B9++EFuuOEG0zxPD4oaxMaPH2+a6mmCpAfWjRs3yo033pjjcj/66COZOXOmbN++XWbNmmWm3Xzzzflap9WrV8vhw4dNcqOBRIONLksft23bZhKhu+++W3766SdZsGCBvPbaaya4Kg1mWvaDDz5o1m3o0KFmugaajHr37m0SsJiYGJM06jpqEjlp0iRb++/SpUtm/7Rv314mT54s8+fPN/tSg864ceOkf//+Zl1nzJhhmjfedNNNplyl26jNDzVI6zQNEhoMNVH9/vvvzQ8L9d5778k///lP+Z//+R8ZOXKknD9/3gRcTXD79euX43odOnTIfEZXXnml2Z+O/QMAnkArtbRpdufOnWXYsGHy448/yjvvvCM7duwwFXUlS5Y082kFnMYCTRr0eK3Hyf/93/818+zatctUhuVEl9OtWzfzY33ZsmUmKcoPfZ82G+/bt6/UqFHDJFC6XpqM6XFZk4CMHnvsMZMUaHzUeV9//XUTAxYtWuScR2PSwIEDzfpojNFKO11m27ZtzTZoRWZ+Y58juenatauJd7of09LSTPlVqlQp0GdBHIPLWICPCQ0NtYYPH57ja+np6Va9evWsbt26mXGHP//806pVq5bVpUsX57Q5c+ZY+hU5cuSIc9rAgQOtsmXL2l4nXX5WCxYsMMvfsGGDc9qUKVOylemg5Wr5WY0fP968Z/DgwZmm33XXXValSpVsracuX5c1ceJE57QzZ85YpUuXtgICAqyFCxc6p+/fv9/Mq+U7nD9/3rp06VKmZeq2BAcHWxMmTHBO69Gjh9W4ceNc18WxXSdPnrR++OEHKywszGrVqpV1+vRpW9sEAMUhY4z47bffrFKlSlldu3bNdAx88803zTyzZ882zy9cuGBdffXVVpMmTay//vrLOd/nn39u5nvuuedyjDebNm2yQkJCrNtuu80cZwsbf7Zu3WrKmzt3brbt6dy5c6b4+MQTT1glSpSwzp49a56npKRYFSpUsB566KFMyzxx4oSJvxmn5zf29ezZ07riiiusY8eOOad9//33ply7P1WJY3AlmvbB52htntYIJSYm5tgs4sCBA6a2SM9UaZekOmizuU6dOsmGDRskPT29yNcpY82h1lppmW3atDHPMzY5LOyZuIy0yYVuY3Jysu1l6dmvjPtTm4RoTZ6e9XLQafqa1t45aHMWPcPnqBHU8rVphM6bcTv1fQkJCaamNC/79u0zNYFaw7lmzRpTUwoAnkSPTRcuXDDNyh3HQPXQQw+ZpmH/+c9/zHPtWlyvtdWmdtqEzUGbxV133XXO+TJat26dOfOjMWrJkiXmOFvQ+HPx4kVzXNamZXoczin+aKuHjF2OayzR47k2SVN6JkWbK957773OGKqD3hKkdevWZn3txD5d9sqVK6Vnz55yzTXXOOdv2LCh2e6CIo7BFUik4HP0NL4etMLDw01TOG0m4DhIahKltEmCNiHIOGhTOG0bnpSUVOTrpG2u9dS/NlPQwKLlOZoRFFV5GQOQchyoz5w5Y2s5Gtx1/TLSa660SUjW+3no9IzL1yRUmyXWq1fPBCNttqDL0uYOGbfz6aefNoFJPx+dV68juNw1anfccYdpf6+BVn+QAICncSQZ+mM7I+2YSK8Pcrx+ufmUJlKO1zMmH5pktWjRQhYvXlygjo7++usvee6550xMzHhc1mQop/iTVyxxxFFtopY1juotQjRRtBP79PorXUeNBVnltJ/ygzgGV+EaKfgcrW3SGrSlS5eag7reQFfbcGtNnuNsk0673M10i+MGiLpO2kZdO5PQcrUMXRft8KKozoBpbWBOLMsqkuXkZ/naRfuzzz5rLiZ+8cUXTTtwrdnTWtqM26k1jXr9wOeffy5ffvmlfPLJJ+YibQ32eo1BRvfcc4/pZEPbuD/88MO2tgUAvJn+kNfbbOg1UXqs1M4T7NJrnvQ6YD0O67VAmjhoMqHXTOUUf/I61jveo9dJ6XVPWQUFBbk09uWEOAZXIZGCT6pWrZppOqGD1o5pJxMvv/yyqWVSWiOkFwS7gtZ0rV271hxY9QDr4KjVyyi3O7h7w93dtZOPjh07mvtsZaQ1n1kvqtUmFn369DGDNonRC3/1M4qOjs7U5EWTXg3M+llqjd7lLuIFAHfR3t+U/rDWM1AOemw7cuSIM95knE/P6GSk0xyvZzzu64/vHj16mM4PtKc87STC7nFZW2FMnTo105munHoTzA9HR0famVFucTS/sU/P9ujZqpxiou4TVyOOwQ6a9sGnaHvmrE0V9GCvvexosz3tqU+DwKuvvppj17E5dfFaWI4asKxnhrQnpKz0oKxyCnD6WkEDn6votmbdTu3mN2u3vlm7vtXmKtrDor5X2/Bn/SGhvTxpz0j6Y2D58uXFuAUAYJ8mFHoce+ONNzIdA/XHuMYkbZ6ntMc9jUnaW5zGJAdNkLRnWcd8GelytUWF9vKnTcS059jCHpe1226NlwWh1y1pZaSeucl6vM4YR/Mb+3Q+Xab2lKddljvo/tCmcK5GHIMdnJGCz91DSttA68Hq+uuvN80I9MJOvRhUa+P09LxeC6Xdouq9H7RLVr2/hB4g9QJZDQ6fffZZka6TLtPRBaseXLU8bXKotZRZaaKntHtWbXah3eVq4NQkSl/TbZk2bZpJDLWduV7Y60m02cmECRPMftWu4bVLW61NzVhDq7SbW20SEhkZadrOa8B88803zY8Ira3LSj+3efPmmYuRtanIF198ka02FwDcRc+q6FkIPfuizdbuvPNOczZFm3ppAqT3iVJ6TNem5nqM1M4HtMMGR/fn2hHBE088kePy9YyNNiHT457Gr/Xr15v7EOX3uKzN8LRJn/7Q37p1q4klej+pgsY07er8/vvvN609NFbp9msSpJ1l6HFdj+d2Yp/uN20ep83y9ayNdn/uuEeTXpvkSsQx2OLSPgKBYpaammqNGTPGuv76663y5cubrmN1/O233840365du6y7777bdA+uXZpee+21Vu/eva21a9cWS/fnCQkJpjty7TJWu4ft1auXlZiYmK3bVfXiiy9a1atXtwIDAzOVr920tm/f3nThqtMdXaFn7F41o5zWPy+X274OHTrk2M2r7jftjjdjt7GjRo2yqlWrZtYzMjLSdLOr79fB4d133zXb4tj/derUMZ9bUlKSc56ctku70tXllCtXztq2bVu+twsAilpOx1jt7vy6666zSpYsaVWpUsUaNmyY6Xo7q0WLFlktWrQwx78rr7zS6t+/v4kTeR2PT506ZTVq1MiqWrWqdeDAgXytp5b/wAMPWFdddZU5durtPzSe6PE74y01HNuzY8eOTO9ft26dma6PWafrsjSmadflehwfNGiQtXPnzgLFvvXr11sRERGmG/natWtbM2bMcMYBO4hjcKUA/WMv9QIAAAAA/8Y1UgAAAABgE9dIAYWgFxHr/S9yk1P3sK7mLesJAMgf7TApp06TMtJrly7X5be3IY7BE9G0DyiEQYMGmXtD5MYTvmLesp4AgPzRm81nvV9RVtqxg3Zi4QuIY/BEJFJAIXz//feSmJiY6zyuul+VL6wnACB/Dh8+bIbctG3bNtP9jLwZcQyeiEQKAAAAAGyiswkAAAAAsIlECgAAAABsIpFCvnzwwQfmbuiXu+BV77ieH7GxsTJ69GjxB3Fxcebu7/7q6NGj5i72qmXLluIP9u3bZy6IBuAaxCb7iE3EJhQdEqnLHFDDw8Nl5syZEhUVJe3atZP27dtLv3795NKlS2aevXv3SqdOnaRDhw5y++23S3x8vJm+Z88eM69Ov/nmm+X48ePmAsnmzZvnepDOWGbWL7bjuX4Junfvnm16xgCgASUyMlLOnj0rAwYMkBo1ahTJPtGyb7rpJilO6enpxbZsd+xf/cyHDRsm3qwwn0nGYFUcPP0746vHQn3UQbsi1v+PZ555xkzXi9rfeOMN53t1/+p+1ekvvviimfbUU09JhQoV8uyyGZ4Rl7zhe0ZsIjbZRWzyfrEeFJtIpC6jT58+MnToUDO+YsUK2bBhg5QrV878Y1+8eFHuu+8+8wGuX79eoqOjzXOlH4rW9Oj0tWvXSqVKlaRRo0by+uuv2yrzchISEkxQzIkG0ZEjR8qSJUvMP8TcuXPzvKdCWlqa9O7d2/R0M2TIEPPlzvjFd4xnrNl7+eWXTeDSf1gt0+HYsWPSrVs385gbXf877rhDWrVqZd5/ww03mPW+//77zfbpumjQHzFihJm/f//+pqce3Z916tQx07TL13Xr1pn10vfdeuut5kdCbveYcPX+dRwQL7c/9X9GD6Q66P+M7k9dR0cta8+ePc126YFAf/icPn3azNOxY0fp0aOH5EV/XGkZul9uu+02mTJliixatMi8dujQIbn33nvN+MSJE808us8dn2fGz6SgdJu0PF3nc+fOycCBA00Anz9/vnlde5vS/xd9/YknnihQGe74zuT2HXrttdfMtIULF0rr1q2lTZs2snLlSjNNA3eLFi2kV69eZl9rMPfGY6H+X+sQGhoq77//vvm/3Lhxo5mm27pmzRr57rvvzHFg8+bNsmnTJud3efLkyeZ/AN4Tl7KWeznEJmITsem/iE3+E5tIpGxISUmRkJAQ2bZtm9nhjgOnZraa/WrtX+nSpc2HpV9OHS/qbkf14KcfeE73ihg8eLAsXrxYqlSpku/lffrpp1K3bl2zzho88rJ7927Zvn27bNmyxfxzNm7c2Ew/ePCg+YeeM2eOXHvttbku488//5Tly5ebA8O4cePkzJkz8thjj5mD2CuvvGK2UX8gaODRR6010C+DDg0aNDAH7q+//tocCFS9evXkiy++MM9Xr14tnrR/c9OwYUPz5a9YsaJcuHDB7E99dHRnW6ZMGbNduo8mTZoku3btkhtvvNEE6aVLl+a5fJ1Ha7D0x1Pfvn3Nftf1VxpE9CCkp/t//PFHM48eYLUGR2X8TApKazy1DN2uEydOyPTp083n6agZGjt2rLz99tvm9fPnz8vOnTulOBT3Z5r1O6Q/EmJiYsw+1QCln5967rnnzA+uefPmOc8UeDv9nxkzZowZDwoKkieffFIWLFhgjnsHDhyQH374wbym/+Pw3bikiE3EJmKTPcQm34hNJFL5oLUymq1r7YEeYLQGKiwsLNM8elDQ6Vqzoh/Q9ddfb76oGriKUkREhJw6dSpbzZp+CbSWwe6N9zTI6DJVTsEqa+/4+/fvN7VQAQEB5nlg4N//Qrrd+sXPul9yovtS36/78pdffjH/yPpld6yPYz30Uf/htTw9yOlBVWsmdVu1pkV/EDiWp/Q0rx5kPWn/5rY/mzVrZh51nznGq1ev7tyGjJ+L7getmStbtqypBZ02bVqeZWXdl/pcT3knJyeb2hn9v9bmPfrDQ2vetImQ47R2xs+kKNSuXdv82NPB0QxJ/5f089Sy9QeQfr+KQ3F/plm/QydPnpRrrrnGHLB1e0uWLGn+X3W7r7zySgkODpYmTZqIN9L/Gf28HE1Ssh4LHcdB/TGvP0YeffRRqV+/vixbtsyNa+2bPCkuKWITsYnYZA+xyTdiE4lUPmjNjNa46GnPqVOnSrVq1bLdFE6/aPqhae3BjBkzzD+w1kZ99NFHtstzBAKltSGOg7LDqFGjzHpkpIFCa8Nmz55tqyw9IOm2KUetS4kSJUwtpw5Zb/anAUZPhzoOuo52ynrw1Bq/y130m/VCV32/Bh/dl46A51gfPXCpHTt2mH2oNYv6nlKlSpnTzlprpKf3c9pf+bktmiv3b277M+N65LQNGT8X3S/adGf8+PGmJk5rk37++edcy81pX2qTDK1B1OChB83rrrvOBEHHKfEvv/zSzJ/xMykoPUg7AlPG7XPQGly9S72Wq9uo13QUlKs/09y+Q5UrVzaBUddDfxhoTa7WiOn/gf4Q0efavMAbOZpP6KPKeix0HAeV1jRrDbX+0PzXv/7ltnX2Va6OS4rYRGxSxKb8Izb5fmwikbJBa0J+++03c5r+22+/NW15lba1dNQ6ae2Mg/7TFuR+x7Vq1TIHZ6WBoWnTpple79Kli/lyaPtPBz246EFs1qxZti6i1IOX1r7oBcqOMocPH25q2vQLnrUWT2untC21tkPX9tCOL53WRulpaD1F7PjiXo62X9V26NpG+qWXXsr02tNPP21qELV8R3DSA5G26deaFd2nWpuq0wvKlfs3r/2ZGz2o3XLLLeb6Br0QUgOOLkeDi+6HvC481c9WT9PrvtJT2toWWH90vfrqq8727vp5ahDTZernqfu+qOh+/eabb0yZerFsVho0H3nkEVOu7vO87ljvSZ9pbt8hDUpa46X7vWvXrs7/8QkTJph5tP2/tnXXYF4ctKmK/qhxBQ1I+v+ktGZTf7TqNN3Pv//+u5mu7fuLa1vhurikiE3EJkVsyj9ikx/EJgvZrFu3zho1apQZ79Chg9W2bVvz2K5dO+vw4cNmelxcnNWxY0erffv2Vvfu3a2jR4+a6RMmTLBuvPFGM3+PHj2slJSUbMvMq8wDBw5YnTt3tqKiosyyjx8/bqYPHDjQ2rt3rxlfuHChRsJs7z1x4oTVrFkza/fu3eZ5REREvrdbl61l+CJ37N8vv/zSevbZZwu0vnPmzLGmT59eoPf6C0/4zthx4cIF83j+/HmradOmVlpamuWtx0Idjh07ZrZh7NixVmRkpHXzzTdbr732mplXj5N6vNT527RpYy1evNi5TH2v47gIz45LWechNhU9YpPv8YTvjB3EpsLFJhKpHGzdutX887777rtFsrzvvvvOat26tfXyyy+7rEx1//33W61atcr3/L4crFy9f/XLql/enTt3FmjZBKu8ecJ3xg49YOuBukWLFtb7779v+eM+HjNmjNWgQQPr3LlzRbI8f+KOuFQc5Spi038Rm3yPJ3xn7CA2WYWKTQH6p/DntQAAAADAf3CNFAAAAADYRCIFAAAAADaRSAEAAACATSRShZCamirPP/+8efTlMt1VLtvqe2W6q1x/KdPfthU586f/AX/ZVvav75XprnL9pUxXlUtnE4WgNzTT+07oHbn1LtG+Wqa7ymVbfa9Md5XrL2X627YiZ/70P+Av28r+9b0y3VWuv5TpqnI5IwUAAAAANpFIAQAAAIBNQeLn0tPTJTExUcqXLy8BAQG2TxlmfHQFd5TprnLZVt8r013l+kuZ7iq3MGVq6/KUlBQJCwuTwEDq9hyITZ5brr+U6a5y/aVMd5XrL2UWplw7ccnvr5FKSEiQ8PBwd68GAPi1+Ph4qVGjhrtXw2MQmwDA8+OS35+R0to+1VZulSAp6e7VAQC/kiYXZZN84TwWw72xaelPe11WFgB4ouQ/0uXaG47mKy75fSLlaDKhgSoogEQKAFzq/7eJsNt8zde5KzaFlKd5JQDkNy5xxAQAAAAAm0ikAAAAAMAmEikAAAAAsIlECgAAAABsIpECAAAAAJtIpAAAAADAJhIpAAAAALCJRAoAAAAAbCKRAgAAAACbSKQAAAAAwNsSqdjYWAkPD5eZM2dKVFSUtGvXTtq3by/9+vWTS5cumXn27t0rnTp1kg4dOsjtt98u8fHxZvqePXvMvDr95ptvluPHj8v3338vzZs3l9GjR7t5ywAA3orYBADw+ERK9enTR4YOHWrGV6xYIRs2bJBy5crJ1q1b5eLFi3LfffeZYLZ+/XqJjo42z9WLL74o77zzjpm+du1aqVSpkjRq1Ehef/31y5aVmpoqycnJmQYAALIiNgEAPD6RyklKSoqEhITItm3bTC1enTp1zPTIyEhJT083NX+lS5eWNWvWyLlz58z4FVdckedyY2JiJDQ01DlojSMAAPlBbAIAeGwi1b17d2nRooUkJCRIw4YNJTExUcLCwjLNU6NGDTN9ypQp8sMPP8j1119vag41aOVFaw2TkpKcg6MpBgAAl0NsAgB4fCKlzSd27dolvXr1kqlTp0q1atVMYMpIA5kGsCpVqsiMGTPk4MGDUq9ePfnoo4/yXH5wcLCpTcw4AACQG2ITAMDjEymHihUrym+//SZt2rSRb7/9Vg4dOmSmb9682Txqs4cDBw44569cubJYluW29QUA+D5iEwDAIUg8sPlEiRIlTFvzDz/8UEqVKiXz5s2Thx56yPSUVLZsWfNcLVy4UD7//HPTBr1ChQrO6QAAFCViEwDA4xIpvQh39erVpucj7W42J9rO/Kuvvso2/dlnnzVDRtrF7NixY+XOO+8stnUGAPg2YhMAIC8Blp+3OdAuZrWHpCjpIUEBJd29OgDgV9KsixIry0wHC1wX5P7YtDIxzmVlAYAnSk5Jl4r1D+crLnnsNVIAAAAA4KlIpAAAAADAJhIpAAAAALCJRAoAAAAAbCKRAgAAAACbSKQAAAAAwCYSKQAAAADwthvyAgAAz9AtrLlbyuX+VQC8EWekAAAAAMAmEikAAAAAsIlECgAAAABsIpECAAAAAJtIpAAAAADAJhIpAAAAALCJRAoAAAAAbCKRAgAAAACbSKQAAAAAwCYSKQAAAACwiUQKAAAAAGwikQIAAAAAb0ukYmNjJTw8XGbOnCktW7bM9Jrj+aBBg6R79+7Zput7R48ebca3bt0qkZGRcvbsWRkwYIDUqFHDpdsBAPAdxCYAgMcnUqpPnz4ydOjQXOdJSEiQPXv25Pja3r17ZeTIkbJkyRKpUKGCzJ07V6pWrZrjvKmpqZKcnJxpAAAgK2ITAMDjE6n80Nq9yZMnZ5t+5MgRGTx4sCxevFiqVKmS53JiYmIkNDTUOWiNIwAABUFsAgD/5TWJVEREhJw6dUqOHTuWafratWuldevWUrNmzXwtJzo6WpKSkpxDfHx8Ma0xAMDXEZsAwH95VCIVEBDgHD9//ryULl060+ujRo2SqVOnZpqmNX7Hjx+X2bNn56uM4OBgCQkJyTQAAHA5xCYAgMcnUrVq1ZK4uDgzvmnTJmnatGmm17t06SK7du2S06dPO6cFBgbK/PnzZdasWbJq1SqXrzMAwLcRmwAAOQkSDzJx4kQZNmyYpKWlmRo/DUBZjRgxQvr27ZtpWpkyZWTp0qXStWtXcyFvs2bNXLjWAABfRmwCAOQkwLIsS9xo27Zt8vDDD8vw4cPz7B0pv7SL2f3798v27dvznFd7RtILe6OkhwQFlCyS8gEA+ZNmXZRYWWauC/Kk5mzEJtdamfj3GT8AcLfklHSpWP9wvuKS2xMpd/O3YAUAnsRTEyl387fYRCIFwBsTKY+6RgoAAAAAvAGJFAAAAADYRCIFAAAAADaRSAEAAACATSRSAAAAAGATiRQAAAAA2EQiBQAAAAA2kUgBAAAAgE1Bdt8AAABQlLqFNXd5mdwEGEBhcUYKAAAAAGwikQIAAAAAm0ikAAAAAMAmEikAAAAAsIlECgAAAABsIpECAAAAAJtIpAAAAADAJhIpAAAAALCJRAoAAAAAbCKRAgAAAACbSKQAAAAAwNcTqQ8++EC2bt2a42vPP/+8fP755y5fJwCA/yIuAYB/ChIvM2jQIHevAgAATsQlAPBPHnNGKi0tTXr37i2dO3eWIUOGmMDUsmVL5+uO8Yy1ey+//LLcdNNNEhUVJXv37nXOe+zYMenWrZt5zCo1NVWSk5MzDQAAuCsuKWITAHgfj0mkPv30U6lbt66sWbNGWrVqlef8u3fvlu3bt8uWLVskNjZWGjdubKYfPHhQhg4dKnPmzJFrr7022/tiYmIkNDTUOYSHhxfL9gAAvJur4pIiNgGA9/GYREoDTUREhBnPKWBZlpXp+f79+6Vdu3YSEBBgngcG/r0pU6ZMkcGDB0tYWFiO5URHR0tSUpJziI+PL4atAQB4O1fFJUVsAgDv4zGJlNb67dq1y4zv3LnTPJYoUUJSUlLMcPjw4UzzN2zYUDZt2uQMZOnp6eZx2rRpptbvchf+BgcHS0hISKYBAAB3xSVFbAIA7+MxiVTPnj1NbV6nTp0kLi7OTBs+fLip3Rs1alS2mrxmzZqZ9unaFr1jx47y3Xffmelly5aVxYsXy3PPPecMgAAA2EVcAgDkJsDK2jbBA+zbt09effVV06VscdMLerU9epT0kKCAksVeHgDgv9KsixIry0xzNk8+C+PKuKSITcVvZeLfyTEAZJScki4V6x/OV1zymDNSAAAAAOAtPDKRatKkictq/QAAyAtxCQDgFYkUAAAAAHgyEikAAAAAsIlECgAAAABsIpECAAAAAJtIpAAAAADAJhIpAAAAALCJRAoAAAAAbAqy+wYAAABv1y2suVvKXZkY55ZyARQ9zkgBAAAAgE0kUgAAAABgE4kUAAAAANhEIgUAAAAANpFIAQAAAIBNJFIAAAAAYBOJFAAAAADYRCIFAAAAADaRSAEAAACATSRSAAAAAGATiRQAAAAAeEsiFRsbK6NHj3ZX8QAAZENsAgB49Rmp9PR0d68CAACZEJsAAB6TSO3Zs0fuuOMOadWqlezdu1duuOEGGTlypNx///2SkJAgnTt3lvbt28uIESPM/P3795fExERZu3at1KlTx0x74YUXZN26dfL888+b9916663SoUMH+euvv3IsMzU1VZKTkzMNAAA4EJsAAB6fSP3555+yfPlymTt3rowbN07OnDkjjz32mMyfP19eeeUV07xiw4YNJvDoY9u2bWXjxo1maNCggRw/fly+/vpradOmjVlevXr15IsvvjDPV69enWOZMTExEhoa6hzCw8NdvNUAAE9GbAIAeHwi1aJFCwkICJCGDRvKL7/8IhUrVpS6deua1w4ePGhqA5U+HjhwQNq1a2eC1o8//ihDhgwxtX9paWlSunRp5/KUBiANfDmJjo6WpKQk5xAfH++y7QUAeD5iEwDA4xOpuLg4sSzLBJ9q1apJYOB/V0eD1vbt2834jh07TI1e48aNzXtKlSplmlW88cYbpsmFgwY+B11uToKDgyUkJCTTAACAA7EJAODxiZQ2X9B26Pfdd5+89NJLmV57+umnZcqUKaamzxGcNBhVqlRJIiIipHLlynLu3DkzHQCAokJsAgDkR4B1ueoxP6EX9GrQjJIeEhRQ0t2rAwB+Jc26KLGyzDRn4yzMfxGbfNfKxDh3rwKAXCSnpEvF+ofzFZc8svtzAAAAAPBkJFIAAAAAYBOJFAAAAADYRCIFAAAAADaRSAEAAACATSRSAAAAAGATiRQAAAAA2EQiBQAAAAA2kUgBAAAAgE0kUgAAAABgU5DdNwAAAKBguoU1d3mZKxPjXF4m4A84IwUAAAAANpFIAQAAAIBNJFIAAAAAYBOJFAAAAADYRCIFAAAAADaRSAEAAACATSRSAAAAAGATiRQAAAAA2EQiBQAAAAA2kUgBAAAAgE0kUgAAAABgE4kUAAAAAHhbIhUbGyvh4eEyc+ZMadmyZabXHM8HDRok3bt3zzZd3zt69GgzvnXrVomMjJSzZ8/KgAEDpEaNGjmWl5qaKsnJyZkGAAAyIjYBADw+kVJ9+vSRoUOH5jpPQkKC7NmzJ8fX9u7dKyNHjpQlS5ZIhQoVZO7cuVK1atUc542JiZHQ0FDnoIESAICsiE0AAI9PpPJDa/cmT56cbfqRI0dk8ODBsnjxYqlSpUqey4mOjpakpCTnEB8fX0xrDADwdcQmAPBfXpNIRUREyKlTp+TYsWOZpq9du1Zat24tNWvWzNdygoODJSQkJNMAAEBBEJsAwH95VCIVEBDgHD9//ryULl060+ujRo2SqVOnZpqmNX7Hjx+X2bNnu2w9AQD+g9gEAPD4RKpWrVoSFxdnxjdt2iRNmzbN9HqXLl1k165dcvr0aee0wMBAmT9/vsyaNUtWrVrl8nUGAPg2YhMAICdB4kEmTpwow4YNk7S0NFPjpwEoqxEjRkjfvn0zTStTpowsXbpUunbtai7kbdasmQvXGgDgy4hNAICcBFiWZYkbbdu2TR5++GEZPnx4nr0j5Zd2Mbt//37Zvn17nvNqF7PaQ1KU9JCggJJFUj4AIH/SrIsSK8tMBwuedF0QsQm+ZGXi32dUAeQtOSVdKtY/nK+45PZEyt0IVgDgPp6aSLkbsQlFiUQKKJ5EyqOukQIAAAAAb0AiBQAAAAA2kUgBAAAAgE0kUgAAAABgE4kUAAAAANhEIgUAAAAANpFIAQAAAIBNQXbfAAAAAO/RLay5y8vk3lXwB5yRAgAAAACbSKQAAAAAwCYSKQAAAACwiUQKAAAAAGwikQIAAAAAm0ikAAAAAMAmEikAAAAAsIlECgAAAABckUjFx8dLQkKC8/n27dvl8ccfl5kzZxZkcQAAFApxCQDgFYlUv379ZN26dWb8xIkT0qVLFxO0xo0bJxMmTCjqdQQAIFfEJQCAVyRS+/btkxtvvNGML168WJo0aSJbtmyR+fPnywcffFDU6wgAQK6ISwAAr0ikLl68KMHBwWZ8zZo1cuedd5rx6667Tn755Zd8LycuLk7eeeedgqwCAABFHpcUsQkAUGyJVOPGjWXGjBmyceNGWb16tdxyyy1memJiolSqVCnfy2nevLkMGzasIKsAAECRxyVFbAIA5EeQFMCkSZPkrrvukilTpsjAgQPl+uuvN9OXL1/ubFqRH7GxsfL555+bx507d5ppLVu2NOPPP/+8HDx4UH7//XczXWsXFy1aJFWqVDGP2lTj008/lQsXLkhKSoosXLhQSpcuLXfffbcEBARISEiILFu2LFuZqampZnBITk4uyC4AAHiQoopLitgEACi2M1JRUVFy6tQpM8yePds5fejQoaZGsKg0bNhQVqxYIRUrVjRBSYOaPh4+fNi8XqZMGfniiy/MxcQaRHft2mUCpl5wvHTp0hyXGRMTI6Ghoc4hPDy8yNYXAOAeropLitgEAChwIjV+/HjTzawGkYxq1qwpV199daH2rGVZzvFmzZqZx7CwMOd49erV5cyZM2Y8IiLCPLZq1UoOHDggHTp0kLJly0r//v1l2rRpOS4/OjpakpKSnIN2mQsA8G7FGZcUsQkAUCSJlDZLqFOnjnTq1Ek+/vjjTM0RCqJEiRKmCYQOjho9pc0gchp3BDSt5VPa3KJu3brmYmMNptpL06pVq+Tnn3/OVpZejKxNKzIOAADvVtRxSRGbAABFnkhpj0Y7duwwF/eOHDlSqlatai7M1WkFMXz4cGnXrp2MGjXK1PDllzal0AuKX3zxRXnqqadM+bocrf2rXLmy1KhRo0DrAwDwLkUdlxSxCQCQmwArY3uFAtCats8++0zmzJkjK1euNF3NDhkyRAYNGmTaeedG59+8eXOBbpaoF/T+8ccfMmLEiEKs/d8X9Op6RkkPCQooWahlAQDsSbMuSqwsM83ZiuosTGHikiI2AYW3MjHO3asAFEhySrpUrH84X3GpQGekMtI8TIOW1sDpuLZPf/PNN82FstqD0eUcOXLEBKkePXoUdhUAACh0XFLEJgBAsZ+R+uabb0xt34IFC0zb7gEDBsiDDz5o2oOr6dOny0svvSS//vqreDJq/QDAN85I+UpcUsQmeDvOSMFbFfsZqaZNm0qbNm1Mzd37779vehd65ZVXnMFK3XvvvXLy5MmCLB4AAFuISwAAr7ghb+/evWXw4MGmu9fLueqqqyQ9Pb0w6wYAQL4QlwAArmb7jJS2O9eLabnrOgDAExCXAABekUiVLFlSzp8/XzxrAwCATcQlAIA7BBb03hqTJk2StLS0ol8jAABsIi4BALziGim9ueDatWvNHdr1At+yZctmen3JkiVFtX4AAOSJuAQA8IpEqkKFCnLPPfcU/doAAFAAxCUAgFckUnqfDgAAPAVxCfAs3cKau6Vc7l8Fj79GSmk79DVr1si7774rKSkpZlpiYqL88ccfRbl+AADkC3EJAODxZ6SOHTsmt9xyi/z888+SmpoqXbp0kfLly5sLffX5jBkzin5NAQC4DOISAMArzkiNHDlSWrZsKWfOnJHSpUs7p991113mYl8AAFyJuAQA8IozUhs3bpQtW7ZIqVKlMk2vWbOmHD9+vKjWDQCAfCEuAQC84oxUenq6XLp0Kdv0hIQE05QCAABXIi4BALwikeratau8/vrrzucBAQHmYt7x48fLrbfeWpTrBwBAnohLAACvaNo3depU6datmzRq1EjOnz8v/fr1kwMHDshVV10lCxYsKPq1BAAgF8QlAIBXJFI1atSQ3bt3y8KFC2XPnj2m1m/IkCHSv3//TBf5AgDgCsQlAIBXJFLmjUFBct999xXt2gAAUEDEJQCAxydSc+fOzfX1AQMGFHR9AACwjbgEAPCKRErv15HRxYsX5c8//zTdzpYpU4aABQBwKeISAMAreu3TGx5mHLQt+o8//iht27blol4AgMsRlwAAXpFI5aRevXryyiuvZKsVdOc9RQAA/svT4pIiNgGA7wgq0oUFBUliYmKhl6M3VRw4cKDEx8dLuXLlJCoqSq655hrp06ePHDp0SJ555hlTwzhx4kRZuXKlWJYlb731ljRt2lRuuOEGadeunZw6dUrmz5+fbdmpqalmcEhOTi70+gIAPFNRxSVFbAIAFDqRWr58eabnGix++eUXefPNNyUyMlIKa+nSpaYr23nz5slHH30khw8flsWLF5tgtWjRIvO4b98+02xj/fr1JkgOGzZMli1bZpp0PPbYY1K3bt0clx0TEyMvvPBCodcRAOA5ijsuKWITAKDQiVTPnj0zPdc7yFeuXFn+8Y9/mJsiFtbBgwelVatWZlwfV61aJUlJSaaGTmv5Ro0aZQLTli1bTI2gKlGihHmsWLHiZQOVio6OlieffNL5XJcZHh5e6HUGALhPccclRWwCABQ6kSruNt4abLZv3y733HOP7Nixw7Rzb926tUyaNElq164twcHBct1110mHDh1k1qxZzh6aVGBg7pd96Xt1AAD4Dldce0RsAgAUOpHKWGuWl2nTphWoZnHJkiXSvn170w5dm1FoMNK26Frbp5o1a2aCmAYsDVBdunSRf/3rX7bLAgB4v+KOS4rYBADIKMDShuQ2dezYUb799ltJS0uTBg0amGk//fSTacKgF9Q6Fx4QIF999ZV4Mm0+ERoaKlHSQ4ICSrp7dQDAr6RZFyVWlpkmciEhIQVeji/FJUVsAgpmZWKcu1cBXi45JV0q1j+cr7hUoDNSd9xxh5QvX14+/PBD0+5b6YW0DzzwgOmVSNuJAwDgKsQlAIBXnJGqXr26uci2cePGmaZrb0Vdu3Ytsq5mXYFaPwDw/jNSvhSXFLEJKBjOSMGVZ6QCC3qAP3nyZLbpOi0lJaUgiwQAoMCISwAAVytQInXXXXeZ5hJ60W1CQoIZPvnkExkyZIjcfffdRb+WAADkgrgEAHC1Al0jNWPGDBk9erT069fP2bWr3j1eA9aUKVOKeh0BAMgVcQkA4BXXSDmcO3dODh06ZMbr1KkjZcuWFW9DO3QA8P5rpHwpLiliE1AwXCMFj++1z0EDlN4zAwAAT0BcAgB49DVSAAAAAODPSKQAAAAAwCYSKQAAAACwqVDXSAEAAACeoltYc5eXSQcX/oszUgAAAABgE4kUAAAAANhEIgUAAAAANpFIAQAAAIBNJFIAAAAAYBOJFAAAAADYRCIFAAAAADaRSAEAAACATSRSAAAAAGATiRQAAAAA2EQiBQAAAAC+mkgdPXpUVq1aZcZbtmzp7tUBAIDYBAB+zCsTqcJITU2V5OTkTAMAAAVBbAIA/+U1idQ777wjixYtkqioKDl37pwMHDhQmjdvLvPnzzevHz58WLp162Zef+KJJy67nJiYGAkNDXUO4eHhLtwKAIAvITYBgP/ymkRq2LBh0qdPH4mNjZUTJ07I9OnTZcOGDfLGG2+Y18eOHStvv/22ef38+fOyc+fOHJcTHR0tSUlJziE+Pt7FWwIA8BXEJgDwX0HihWrXri0hISFm/NKlS+Zx//79MmTIEDOekpJiagBzaq8eHBxsBgAAihKxCQD8i9ckUiVLlnQGpoCAgGyvN2jQQF599VW59tprxbIs57wAABQXYhMA+C+vadrXtGlT+eabb6RXr15y9uzZbK9PmjRJHnnkEenYsaN06dJFEhMT3bKeAAD/QWwCAP8VYGkVmR/TnpH0wt4o6SFBASXdvToA4FfSrIsSK8vMdUGOZnEgNgHeZGVinLtXAUUoOSVdKtY/nK+45DVnpAAAAADAU5BIAQAAAIBNJFIAAAAAYBOJFAAAAADYRCIFAAAAADaRSAEAAACATSRSAAAAAGATiRQAAAAA2EQiBQAAAAA2Bdl9AwAAAIC/dQtr7pZyVybGuaVc/BdnpAAAAADAJhIpAAAAALCJRAoAAAAAbCKRAgAAAACbSKQAAAAAwCYSKQAAAACwiUQKAAAAAGwikQIAAAAAm0ikAAAAAMAmEikAAAAAsIlECgAAAABsIpECAAAAAG9LpGJjYyU8PFxmzpwpLVu2zPSa4/mgQYOke/fu2abre0ePHm3Gt27dKpGRkXL27FkZMGCA1KhRw6XbAQDwHcQmAIDHJ1KqT58+MnTo0FznSUhIkD179uT42t69e2XkyJGyZMkSqVChgsydO1eqVq2a47ypqamSnJycaQAAICtiEwDA4xOp/NDavcmTJ2ebfuTIERk8eLAsXrxYqlSpkudyYmJiJDQ01DlojSMAAAVBbAIA/+U1iVRERIScOnVKjh07lmn62rVrpXXr1lKzZs18LSc6OlqSkpKcQ3x8fDGtMQDA1xGbAMB/eVQiFRAQ4Bw/f/68lC5dOtPro0aNkqlTp2aapjV+x48fl9mzZ+erjODgYAkJCck0AABwOcQmAIDHJ1K1atWSuLg4M75p0yZp2rRppte7dOkiu3btktOnTzunBQYGyvz582XWrFmyatUql68zAMC3EZsAADkJEg8yceJEGTZsmKSlpZkaPw1AWY0YMUL69u2baVqZMmVk6dKl0rVrV3Mhb7NmzVy41gAAX0ZsAgDkJMCyLEvcaNu2bfLwww/L8OHD8+wdKb+0i9n9+/fL9u3b85xXe0bSC3ujpIcEBZQskvIBAPmTZl2UWFlmrgvypOZsxCYAnm5l4t9nylG0klPSpWL9w/mKS25PpNyNYAUA7uOpiZS7EZsA5IVEyv2JlEddIwUAAAAA3oBECgAAAABsIpECAAAAAJtIpAAAAADAJhIpAAAAALCJRAoAAAAAbCKRAgAAAACbSKQAAAAAwKYgu28AAAAA4F7dwpq7vExuApwZZ6QAAAAAwCYSKQAAAACwiUQKAAAAAGwikQIAAAAAm0ikAAAAAMAmEikAAAAAsIlECgAAAABsIpECAAAAAJtIpAAAAADAJhIpAAAAAPDlRGrfvn0yaNAgd68GAABOxCYA8E9elUgBAAAAgCfw+EQqLS1NevfuLZ07d5bXXnvNTFu4cKG0bt1a2rRpIytXrjTTVq1aJS1atJBevXpJ+/bt5ejRozkuLzU1VZKTkzMNAADYQWwCAHh8IvXpp59K3bp1Zc2aNdKqVSu5dOmSxMTEyPr1602AGjdunJnvueeek7Vr18q8efMkPj7+ssvT94aGhjqH8PBwF24NAMAXEJsAAB6fSB08eFAiIiLMuAarkydPyjXXXCNXXHGFhISESMmSJU3NoAaxK6+8UoKDg6VJkyaXXV50dLQkJSU5h9wCGwAAOSE2AQA8PpHSGr9du3aZ8Z07d0rlypXl2LFjcv78edP04cKFCxIUFCQlSpSQM2fOmOfffffdZZenwUyDXMYBAAA7iE0AgCDxcD179jTtzjt16iT169c3QWns2LGmrXlgYKC89NJLZr4JEyaYeWrVqiVVq1Y1tYEAABQHYhMAIMCyLEt8wMWLF02A0gt2tZmF1hRqYMuL1hxqe/Qo6SFBAQQ4AHClNOuixMoy05zNF8/CEJsA+JKViXHi65JT0qVi/cP5ikse37TPzoW/UVFRctNNN8njjz+er0AFAEBxIjYBgO/y+KZ9+aVdy+oAAICnIDYBgO/ymTNSAAAAAOAqJFIAAAAAYBOJFAAAAADYRCIFAAAAADaRSAEAAACATSRSAAAAAGATiRQAAAAA2EQiBQAAAAD+ekNeAAAAAMWnW1hzt5S7MjFOPBFnpAAAAADAJhIpAAAAALCJRAoAAAAAbCKRAgAAAACbSKQAAAAAwCYSKQAAAACwiUQKAAAAAGwikQIAAAAAm0ikAAAAAMAmEikAAAAAsIlECgAAAABsIpECAAAAAJtIpAAAAADApiDxM6mpqWZwSE5Oduv6AABAbAIA7+N3Z6RiYmIkNDTUOYSHh7t7lQAAfo7YBADex+8SqejoaElKSnIO8fHx7l4lAICfIzYBgPfxu6Z9wcHBZgAAwFMQmwDA+/jsGakTJ07I+PHj3b0aAAA4EZsAwHf4bCJVtWpVeeGFF9y9GgAAOBGbAMB3+GwiBQAAAADFhUQKAAAAAGwikQIAAAAAm0ikAAAAAMAmEikAAAAAsIlECgAAAABsIpECAAAAAJtIpAAAAADAJhIpAAAAALCJRAoAAAAAbAqy+wYAAAAAcJVuYc1dVlaadVFEDudrXs5IAQAAAIBNJFIAAAAAYBOJFAAAAADYRCIFAAAAADaRSAEAAACATSRSAAAAAGATiRQAAAAA2EQiBQAAAAA2kUgBAAAAgE0kUgAAAABgE4kUAAAAAHhjIhUbGyvh4eEyc+ZMiYqKknbt2plHHZKSkiQ9PV2eeeYZM71t27byxhtvON87evRoiYyMNNNffPFFM+2pp56SChUqyB9//OHGrQIAeCviEgAgL0HiIfr06SNDhw6Vjz/+WFasWCHlypVzvvbee+/J6dOnZePGjZKWliY9evSQRo0aSbVq1eTYsWOyefNmM9+ZM2fM4+TJk2X79u05lpOammoGh+Tk5GLfNgCA93FVXFLEJgDwPh5xRiovCxculDFjxpjxoKAgefLJJ2XBggVyxRVXyIEDB+SHH34wr1WsWDHPZcXExEhoaKhz0BpHAADcFZcUsQkAvI9HJlLdu3c3zSf0USUmJkpYWJjz9Ro1aphpderUkbFjx8qjjz4q9evXl2XLluW57OjoaNMswzHEx8cX67YAALxfccYlRWwCAO/jMU37MsrahEKbSmiAqlWrlnmekJDgDGB9+/Y1w4kTJ6RTp06meUVugoODzQAAgCfEJUVsAgDv45FnpLLSgPTqq6+acW2LPm3aNDNN26f//vvvZrpexFuyZEk3rykAwB8QlwAAHnlGSptOlChRwozPnTtXhgwZYnpH0h6QLMuSXr16SZcuXeTIkSMycOBAM00D2bhx49y96gAAH0RcAgB4ZCKlF+euXr3adDOrXc5e7kLcrLRJxYYNG7JN125mtUlFYKBXnHADAHgY4hIAIC8Bllab+THtYlZ7SIqSHhIUQBMMAHClNOuixMoy08FCSEiIu1fHYxCbAMDz4xJVYwAAAABgE4kUAAAAANhEIgUAAAAANpFIAQAAAIBNJFIAAAAAYBOJFAAAAADYRCIFAAAAAN54Q153ctxGK00uivj1HbUAwPXMsTfDsRh/IzYBgOfHJb9PpFJSUszjJvnC3asCAH59LNYb0OJvxCYA8Py4FGD5eTVgenq6JCYmSvny5SUgIMD2nefDw8MlPj4+zzsfFxV3lOmuctlW3yvTXeX6S5nuKrcwZWoI0mAVFhYmgYG0NncgNnluuf5SprvK9Zcy3VWuv5RZmHLtxCW/PyOlO6hGjRqFWoZ+OK78x3BXme4ql231vTLdVa6/lOmucgtaJmeisiM2eX65/lKmu8r1lzLdVa6/lFnQcvMbl6j+AwAAAACbSKQAAAAAwCYSqUIIDg6W8ePHm0dfLtNd5bKtvlemu8r1lzL9bVuRM3/6H/CXbWX/+l6Z7irXX8p0Vbl+39kEAAAAANjFGSkAAAAAsIlECgAAAABsIpECAAAAAJtIpAAAAADAJhIpwENERUXJ448/7u7VAADAidgEXB6JFAAAAADYRCIFAAAAADaRSAEe6j//+Y+EhobK/PnzJT4+Xnr37i0VKlSQK6+8Unr06CFHjx41823YsEFKliwpJ06cyPR+bYrRrl07M37s2DG54447pGLFilK2bFlp3LixfPHFF27ZLgCA9yI2Af9FIgV4oI8//ljuvfdeE6g0SHXr1k3Kly8vGzdulM2bN0u5cuXklltukQsXLkj79u2ldu3a8tFHHznff/HiRfPewYMHm+fDhw+X1NRUE9j27t0rkyZNMssAACC/iE1AZkFZngNws7feekvGjRsnn332mXTo0EHmzZsn6enpMmvWLAkICDDzzJkzx9QAxsbGSteuXWXIkCFm2pgxY8zr+t7z58+bQKd+/vlnueeee6Rp06bmuQY3AADyi9gEZMcZKcCD/Pvf/5YnnnhCVq9ebQKV2r17txw8eNDU+mlNnQ7ahEKD0aFDh8w8gwYNMvNs27bNPP/ggw9MoNKmEuqf//ynvPTSSxIZGSnjx4+XPXv2uHErAQDehNgE5IxECvAgLVq0kMqVK8vs2bPFsiwz7Y8//pCIiAiJi4vLNPz000/Sr18/M8/VV19t2plrzd+vv/4qK1ascDadUA8++KAcPnxY7r//ftN8omXLljJ9+nS3bScAwHsQm4CckUgBHqROnTqybt06WbZsmTz22GNm2g033CAHDhwwAalu3bqZBr3gN2NAWrRokcycOdMsR2v4MgoPD5dHHnlElixZIqNGjZL33nvP5dsHAPA+xCYgZyRSgIepX7++CViffPKJ6d2of//+ctVVV5nekPSC3iNHjpj259okIiEhwfk+veg3JCTENJN44IEHMi1Tl7Ny5Urz3m+//dYsv2HDhm7YOgCANyI2AdnR2QTggRo0aCBfffWVuaN8iRIlTI9GTz/9tNx9992SkpIi1atXl06dOpng5BAYGGjao0+cOFEGDBiQaXmXLl0yvSNpcNP3aK9Kr732mhu2DADgrYhNQGYBlqOxKwCvpz0knTx5UpYvX+7uVQEAwCA2wVdxRgrwAUlJSeZCXb3HB4EKAOAJiE3wdSRSgA/QNurbt283F+x26dLF3asDAACxCT6Ppn0AAAAAYBO99gEAAACATSRSAAAAAGATiRQAAAAA2EQiBQAAAAA2kUgBAAAAgE0kUgAAAABgE4kUAAAAANhEIgUAAAAAYs//A9tOGCBwwJRWAAAAAElFTkSuQmCC"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "['[BOS]', '[UNK]', 'does', 'the', '[UNK]', 'say', '?', '[EOS]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1000x500 with 2 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1AAAAG1CAYAAAD3DRUpAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAQFFJREFUeJzt3QucTfX6+PFnxjCuM3TDNFMhl5SQCX/CIINuk1tCIU4KlQrJq+MoyiQVujJ1kBK5TFdpiNxCumA4RW4xYyiOmplcxmXW//V8z2vv38wYZs2Y2XvtvT/v12uZtddee33X3mavZ57vetZ3BVmWZQkAAAAAoEDBBa8CAAAAAFAkUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIBNJFAAAAAAYBMJFAAAAADYRAIFAAAAADaRQAEAAACATSRQQD5mzZolQUFB8ttvv+VaPmnSJKlZs6aUKlVKGjVq5LX983fPPvus+fyPHDni7V0BANsxorj0799fKlasKJ5+Pz/88IM4LQ74KuKYfyOBAmxaunSpPPXUU9KyZUuZOXOmTJgwoVi3v27dOnPA/euvv855Ttv65JNPirU9AAAAFB4JFGDTihUrJDg4WP79739L37595bbbbiv2BOq5554jgQIAAHAwEijApj/++EPKlSsnZcqU8fauAAAAwEtIoOBXMjMz5fHHH5drrrlGQkND5YorrpAOHTrITz/95F7nu+++k06dOkl4eLiUL19e2rRpI99+++0Ft6t1zFq2d+zYMTOvk9aM25GcnGzq2fXaqbJly0q1atVkwIAB8t///te9jpbujRw50szXqFHD3YbW1+tPbfe9995zL9ftuV6nj3ft2mWWVa5c2byvBx54QI4fP16kmvv9+/fLHXfcYeavvPJKefPNN83zW7dulXbt2kmFChXk6quvlg8//DDX648ePSojRoyQBg0amNeGhYVJ586dZcuWLee09frrr8v1119vPv8qVapIdHT0OdvLa9++fXLttdfKDTfcIL///nuh3hsAlLS33nrLHNc09kRERMjQoUPzrShYsGCBNGnSxHTIXXbZZXLffffJgQMHCtz+5s2b5fLLL5eYmBj5+++/be2THjeHDBkidevWNe1deuml0qNHj/Neu5WVlSVPPvmkaUeP9V26dJHDhw+fs96SJUukVatWZp1KlSrJ7bffLv/5z38KHftc1q5dKzfffLNZr1atWjJ9+nQpCuIYPCXEYy0BHvDwww/LwoUL5ZFHHpH69eubA7UemH/55Re56aabTBmeHgw1eI0dO9aU5GlipAfUNWvWSNOmTfPd7vvvvy8JCQmyceNGeffdd82yFi1a2NqnZcuWyZ49e0xSowFEg4xuS39u2LDBJEBdu3aVX3/9VebOnSuTJ082QVVpENO2//GPf5h9GzRokFmuASane+65xyRe8fHxJlnUfdTkceLEiYX6/M6ePWs+n9atW8tLL70kc+bMMZ+lBptnnnlG+vTpY/Z12rRppozx//2//2faVfoetcxQg7Mu0+CgQVAT1J9//tn8QaHeeecdeeyxx6R79+4ybNgwOXnypAm0mtj27t073/3avXu3+T+65JJLzOfp+nwAwAm0M0tLsG+99VYZPHiw7NixQ95++235/vvvTQdd6dKlzXra8aaxQJMFPV7rcXLq1KlmnU2bNplOsPzodjp27Gj+SP/0009NMmSHvk7Lw++9916JjIw0iZPulyZhelzWP/5zevTRR00yoPFR150yZYqJAR999JF7HY1J/fr1M/ujMUY763Sbt9xyi3kP2oFpN/a5kprY2FgT7/RzPHPmjGm/atWqRfq/II7BIyzAj4SHh1tDhw7N97ns7Gyrdu3aVseOHc28y/Hjx60aNWpYHTp0cC+bOXOmpV+PvXv3upf169fPqlChQqH3Sbef19y5c832V69e7V42adKkc9p00Xa1/bzGjh1rXjNgwIBcy7t06WJdeumlhdpP3b5ua8KECe5lf/75p1WuXDkrKCjImjdvnnv59u3bzbravsvJkyets2fP5tqmvpfQ0FBr3Lhx7mVxcXHW9ddff8F9cb2vw4cPW7/88osVERFh3XzzzdbRo0cL9Z4AoCTkjBF//PGHVaZMGSs2NjbXMfCNN94w68yYMcM8PnXqlHXFFVdYN9xwg3XixAn3el988YVZ71//+le+8Wbt2rVWWFiYdfvtt5vj7MXGn/Xr15v2Zs+efc77ufXWW3PFxyeeeMIqVaqU9ddff5nHmZmZVuXKla0HH3ww1zYPHTpk4m/O5XZj3913322VLVvW2rdvn3vZzz//bNot7J+pxDF4CiV88Cvae6c9QGlpafmWP+zcudP0DumZKR1aVCctj2vfvr2sXr1asrOzi32fcvYUai+Vttm8eXPzOGdp4cWeectJSyv0PWZkZBR6W3q2K+fnqaUf2nOnZ7lcdJk+p711Llq2omf0XD2A2r6WQOi6Od+nvi41NdX0jBZk27ZtpudPezS//vpr0zMKAE6ix6ZTp06Z8nHXMVA9+OCDpgRs8eLF5rEOEa7X0mpJnZaquWj5W7169dzr5fTNN9+YMz0aoxITE81xtqjx5/Tp0+a4rCVkehzOL/5olUPOocM1lujxXEvPlJ450bLEXr16uWOoTnprj2bNmpn9LUzs020nJSXJ3XffLVdddZV7/euuu86876IijqGkkUDBr+jpej1YRUVFmZI3LQdwHRw1eVJaeqClAjknLXnT2u/09PRi3yetqdZT/FqOoAFF23OVCxRXezkDj3IdoP/8889CbUeDuu5fTnpNlZZ+5L0fhy7PuX1NPrX8sHbt2iYIaXmCbkvLGnK+z1GjRpmApP8/uq5eJ3C+a9DuvPNOU1+vAVb/EAEAp3ElF/pHdk464JBe/+N6/nzrKU2gXM/nTDo0uWrcuLHMnz+/SAMYnThxQv71r3+ZmJjzuKxJUH7xp6BY4oqjWoqWN47qrT40QSxM7NPrq3QfNRbkld/nZAdxDJ7ANVDwK9q7pD1mH3/8sTmY641vtUZbe+5cZ5d02fluglsSNy7UfdIadB0kQtvVNnRfdCCL4jrjpb1/+bEsq1i2Y2f7OtT6mDFjzEXC48ePN3Xe2pOnvbI536f2LOr1AV988YV89dVXsmjRInPxtQZ5vYYgp27dupnBM7SG/aGHHirUewEAX6Z/wOvtMvSaJz1W6qAIhaXXNOl1vnoc1mt9NGHQJEKvicov/hR0rHe9Rq+D0uua8goJCfFo7MsPcQyeQAIFv1O9enVTIqGT9obp4BEvvPCC6VVS2gOkF/p6gvZsLV++3BxQ9cDq4urFy+lCd1z3hbux6+Adbdu2NffJykl7OvNeLKulFD179jSTlr7oBb36fzR69OhcpS2a7GpA1v9L7cE738W5AOAtOpqb0j+o9YyTix7b9u7d6443OdfTMzg56TLX8zmP+/pHd1xcnBnUQEe+08EfCntc1qqLV155JdeZrfxGB7TDNYCRDlJ0oThqN/bp2R09O5VfTNTPxNOIY7CLEj74Da1XzluSoAd5HTVHy/N05D09+L/88sv5DgGb31CtF8vV45X3TJCObJSXHoxVfoFNnytqwPMUfa9536cO15t3eN68Q9hqWYqOmKiv1Rr9vH9A6KhNOtKR/hHw2WefleA7AIDC00RCj2OvvfZarmOg/hGuMUnL8JSOoKcxSUd/05jkoomRjhTrWi8n3a5WUOiofVoKpiPBXuxxWYff1nhZFHpdknZC6pmavMfrnHHUbuzT9XSbOvKdDj3uop+Hlrx5GnEMdnEGCn51DyitcdaDVMOGDU25gF6wqRd5au+bnobXa510eFO9d4MOrar3h9ADo174qkHh888/L9Z90m26hlLVg6q2p6WF2iuZlyZ4SodZ1fIKHfZWA6YmT/qcvpdXX33VJIRaR64X7DqJlpeMGzfOfK46xLsOTau9pzl7ZJUOV6ulHy1btjS18Roo33jjDfPHg/bO5aX/bx988IG5yFhLQr788stzem8BwFv0LIqeddCzLVqedtddd5mzJ1rSpYmP3udJ6TFdS8r1GKmDCuhADK5hzHWAgSeeeCLf7esZGi0V0+Oexq9Vq1aZ+wjZPS5ruZ2W7ukf+OvXrzexRO8HVdSYpkOW33///aa6Q2OVvn9NfnQQDD2u6/G8MLFPPzctg9Pyez1Lo8OYu+6xpNceeRJxDLZ5bLw/oIRlZWVZI0eOtBo2bGhVqlTJDAGr82+99Vau9TZt2mR17drVDPOtQ5NeffXV1j333GMtX768RIYxT01NNcOK69CvOsxrjx49rLS0tHOGT1Xjx4+3rrzySis4ODhX+zrcauvWrc1QrLrcNaR5zmFSc8pv/wtyvvfXpk2bfIdr1c9Nh9XNOfzr8OHDrerVq5v9bNmypRkuV1+vk8v06dPNe3F9/rVq1TL/b+np6e518ntfOiSubqdixYrWhg0bbL8vAChu+R1jddjyevXqWaVLl7aqVq1qDR482AyhnddHH31kNW7c2Bz/LrnkEqtPnz4mThR0PD5y5IhVv359q1q1atbOnTtt7ae2/8ADD1iXXXaZOXbqbTw0nujxO+etMVzv5/vvv8/1+m+++cYs1595l+u2NKbpEOR6HO/fv7/1ww8/FCn2rVq1ymrSpIkZDr5mzZrWtGnT3HGgMIhj8JQg/cd+ugUAAAAAgYtroAAAAADAJq6BAopILw7W+1dcSH7DvHqar+wnAMAeHQgpv8GQctJrk843dLevIY7BaSjhA4qof//+5t4OF+KEr5ev7CcAwB69SXze+w3lpQM26OAU/oA4BqchgQKK6Oeff5a0tLQLruOp+035w34CAOzZs2ePmS7klltuyXU/Il9GHIPTkEABAAAAgE0MIgEAAAAANpFAAQAAAIBNJFBw27Ztm7lQ05N+++03c3dyFR0d7dG2A9V3331n7p6ud5HXO6MDgFN5Iy4pYpPnEZvgS0igcli5cqVERUVJQkKCxMTESKtWraR169bSu3dvOXv2rFln69at0r59e2nTpo3ccccdkpKSYpYnJyebdXV5ixYt5MCBA+aix0aNGsmIESNstZn3IO16rMGjc+fO5yzX17q2vX79enPg+euvv6Rv374SGRkpviBnkCoJgf755qd69eqyYsUKWbdunUydOrVEvjf6UycdejY7O1v++c9/muV6UfNrr73mfq1+vvq56vLx48ebZU899ZRUrlz5gkP05tdm06ZN3dtQ+j0cN26c+/GsWbOkdu3a0q5dO7P+9OnT3c/Fxsba+iPJG+0GSps4P2KT5xGbPI/YRGyKdmCb56WDSOB/vvnmG2v48OFmvk2bNlZmZqaZf/DBB601a9ZYp06dsm688UZr165dZvnatWut1q1bm/nu3btb27ZtM/PHjx+3Tpw4cc42C2qzSZMmuZ5zPe7Xr591ww03WFu2bMm13PXa5ORk6+abb7YOHTp0zmsLcvr0aatHjx5W+/btrQEDBpi25s6dazVt2tRq1qyZ9dVXX5n1vv/+eysmJsa65ZZbrEmTJpllb7/9tmm3bdu2VmJiolUU99xzjxUZGWk+73r16ll9+/a1GjZsaH3wwQfm+d27d1uxsbHm+ccff7zQ2/f255vT+vXrzeeqn+PYsWOtJ554wvz+6LY3bdpk/fHHH9Ztt93mXr9du3ZWenq6VVJWrVpl9enTp8S+Ny4JCQnW4MGD3b9v+h6XLVtmvi/6vXE5evSoez6/7RTUpm5bv58pKSnW/v37ze+1/m66zJw503r99dfN/LFjx8zv1RdffFGo/1NvtBsobeL8Ai02eTsuBVJsclpcUsQmYpMvxCbOQNmQmZkpYWFhsmHDBtNrV6tWLbNceye0B0N7+sqVKydff/21HDt2zMwX99Ch2hvy0ksv5XufhwEDBsj8+fOlatWqhd7uJ598Itdee63Z95tvvtn0ZsbHx8uqVatM79szzzxj1nv66aclMTFR1qxZY577/fffTZv6Ou0xiouLK9L7Gjx4sPTs2dP0Khw6dEhef/11Wb16tbsnSNt96623zPMnT56UH374QUpCSX2+OS1evFjGjh0r33zzjfzrX/+S559/3nyW2hsyadIkc9PDMmXKyMGDB83wtFdccYX5vSsJhw8flpEjR8rkyZOlpM2bN8+0pUJCQuTJJ5+UuXPnmu/Izp075ZdffjHPValS5aLa0W3Xr1/f9LAvXLhQ+vTpI/Xq1ZPt27efs2758uVl1KhRsmjRootq01vtBkqbCMzY5O24FEixyUlxSRGbiE2+8vmSQF2Anjpv3LixpKamynXXXWfuQRAREZFrHT1drsv1QKNftoYNG5qDrgar4tSkSRM5cuSI7Nu3L9fy5cuXS7NmzYp8s7xdu3aZbSsNVHrwuuqqq8wBRA+SpUuXljNnzpgykC5duphTpvv37zeB+cUXX5Rhw4aZMgM92FysmjVrmjZ1cpWl6Bdg4MCBpt2NGzea/4uSUFKfb05Dhw6VL7/80ny5v/rqK/M7o6eTH3vsMff9Le677z5zAJ8zZ45Zr6RosNTta3Asie+N/n+5Sk/yfm9c3xn9Y0//CBkyZIjUqVNHPv3004tq9/jx4+b3VH+P9I+sTp06Sa9evWTBggX5rq/7pH8UXCxvtBsobSIwY5OT4pK/xyYnxSVFbCI2+crnG3LRW/BjS5YskYoVK5oep1deecXUVeqBJic9aOp/hvYATZs2zSzTmtr3339fHn744UK1FxQU5J7XHi3tLcxp+PDhZj9y0h4o7YmaMWOGmS8s7eXbtGmTdOvWzfSg6UFry5Ytpv1Tp06ZSbN7Db6a2YeHh5sAEhwcbNaZOXOmqVeeOHGi2YfC0kDoCkg5379L3bp15eWXX5arr77a3GXctW5ReOPzzUk/uzfeeMN8phoU9fHatWvlxx9/NG2rO++80xzcT58+LaNHj5aSovXArt7qkvre5Kxr16BUo0aNXN8Zde+995pJe3j1+o2i9hjrZ6a/k9qbmJWVZS48123p74zWuo8ZM+ac1+T3R6cvtBsobSJwY5O341IgxSYnxSVFbCI2+UpsIoGyQU/f6gWlzZs3N701u3fvNl/wb7/91jyvF7RpT5d+8ZUe7Ityf2L9Em/evNmUYugBrEGDBrme79Chg7kw7ujRo+5l+oujvUJ6B27tPdEL4grj7rvvNqex9QChPS2lSpUyPS960bFuW0/nK+3V69q1qykLCQ0NlY8//tiUOOjnor+0L7zwghSFvkc9IPfo0cNcBJuXBkAN9hpUdN80WGhPZFF44/PNSUsitNxEe061d1TLJLQ3TH+vXLRUQk89a7v6B0JJ0VKXEydOuHt5S5IGIf1D48033zTv/dVXX5XHH3/cfM76Pbn00kvNhbn6B0txBMYpU6aY8o/u3bubx9qLuGPHjlzr63vXnlbtqb4Y3mg3UNpE4MYmb8elQIpNTopLithEbPKV2EQCVUBmqwdGPTi/99575iCiQ2s++OCDprepQoUK7qE29WD/xRdfmJ4j/cIVZQjOCRMmmIO/fpF1O+++++456zzyyCPmS5+3plMDhx5Aq1WrJjfeeKPtNvVgqD14eenoTjnpwUxLBnLSkU0ulpZEaF15Xq56cj0lq1+Q4uCNzzcnPTDr5OLq3ctLg1S/fv2kJOkp7pL+3qjZs2ebMhft+dbRjDQo6R8k+geB9p7q+9Rl+n/iuq7hYmlts15D4dK2bVtznYD+MakjO+kfC9qTqiNWFefn4I12A6VNBFZs8nZcCqTY5KS4pIhNxCafabNIQ0/4KR2NRkfymD59erFs7z//+Y8ZMeiFF17wWJvq/vvvNyPowPc+Xx0RqHfv3lYgf8YjR4606tata0bL8VSbHTp0sO68884C1/NGu4HSJs6P2OR/fOnz9cW4pIhNJdvu+gBp83yC9J+LT8MAAAAAwP8xCh8AAAAA2EQCBQAAAAA2kUABAAAAgE0kUEWkQ6Q+++yz5idt+ke7gdKmt9oNlDa91W6gtInz4/fd/9r0VruB0qa32uW9+n6bDCJRRBkZGeaGc3qjLh3ulDZ9v91AadNb7QZKm95qN1DaxPnx++5/bXqr3UBp01vt8l59v03OQAEAAACATSRQAAAAAGBTiAQwvYt7WlqaVKpUSYKCggp9ejDnT08IlDa91W6gtOmtdgOlTW+162ttavV4ZmamRERESHAwfXnFEZv4ffe/Nr3VbqC06a12ea++H5sC+hqo1NRUiYqK8vZuAEDASklJkcjISG/vhqMQmwDA2bEpoM9Aae+eukVukxAp7e3dAXABH/+61du7gGKU8Xe2XH3Tb+7jMLwfm/iOAQh0GTZjU0AnUK7SCA1QIUEkUICThVWizMsfFbZ8OhB4KzbxHQMAe7GJoyUAAAAA2EQCBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIBNJFAAAAAAYBMJFAAAAADYRAIFAAAAAL6QQK1cuVKioqIkISFBYmJipFWrVtK6dWvp3bu3nD171qyzdetWad++vbRp00buuOMOSUlJMcuTk5PNurq8RYsWcuDAAfn555+lUaNGMmLECG++LQCAjyIuAQAcfwaqZ8+eMmjQIDO/ZMkSWb16tVSsWFHWr18vp0+flvvuu88EslWrVsno0aPNYzV+/Hh5++23zfLly5fLpZdeKvXr15cpU6act62srCzJyMjINQEA4K24pIhNAOBbvJ5A5SczM1PCwsJkw4YNpueuVq1aZnnLli0lOzvb9PaVK1dOvv76azl27JiZL1u2bIHbjY+Pl/DwcPekvYwAAHgrLiliEwD4FkclUJ07d5bGjRtLamqqXHfddZKWliYRERG51omMjDTLJ02aJL/88os0bNjQ9BZqwCqI9hSmp6e7J1fZBQAA3ohLitgEAL7FUQmUlkps2rRJevToIa+88opUr17dBKWcNIhp8KpatapMmzZNdu3aJbVr15b333+/wO2HhoaaHsScEwAA3opLitgEAL7FUQmUS5UqVeSPP/6Q5s2by08//SS7d+82y7/99lvzU8sbdu7c6V7/8ssvF8uyvLa/AAD/RlwCALiEiMNKJUqVKmXqyd977z0pU6aMfPDBB/Lggw+a0Y8qVKhgHqt58+bJF198YerMK1eu7F4OAEBxIS4BAByVQOkFtsuWLTOjGenQsfnRWvIVK1acs3zMmDFmykmHi3366aflrrvuKrF9BgD4L+ISAKAgQVYA1xjoULE64lGMxElIUGlv7w6AC0hK2+ztXUAxysjMlip19phBE7jmxxmxie8YgECXYTM2OfIaKAAAAABwIhIoAAAAALCJBAoAAAAAbCKBAgAAAACbSKAAAAAAwCYSKAAAAACwiQQKAAAAAHzhRroAAMAZOkY08kq73H8KgK/hDBQAAAAA2EQCBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIBNJFAAAAAAYBMJFAAAAADYRAIFAAAAADaRQAEAAACATSRQAAAAAOD0BGrlypUSFRUlCQkJEh0dnes51+P+/ftL586dz1murx0xYoSZX79+vbRs2VL++usv6du3r0RGRnr0fQAA/AexCQDg6DNQPXv2lEGDBl1wndTUVElOTs73ua1bt8qwYcMkMTFRKleuLLNnz5Zq1aqdd1tZWVmSkZGRawIAICdiEwDAp0v4tDfvpZdeOmf53r17ZcCAATJ//nypWrWqrW3Fx8dLeHi4e9JeRgAACovYBACBy/EJVJMmTeTIkSOyb9++XMuXL18uzZo1k2uuucb2tkaPHi3p6enuKSUlpQT2GADg74hNABC4HJFABQUFuedPnjwp5cqVy/X88OHD5ZVXXsm1THv4Dhw4IDNmzLDdTmhoqISFheWaAADID7EJAODYBKpGjRqyefNmM7927Vpp0KBBruc7dOggmzZtkqNHj7qXBQcHy5w5c+Tdd9+VpUuXenyfAQD+jdgEAMhPiDjAhAkTZPDgwXLmzBnTw6eBJ69HHnlE7r333lzLypcvLx9//LHExsaaC3RvvPFGD+41AMCfEZsAAPkJsizLEi/YsGGDPPTQQzJ06NACRzuyS4eK3b59u2zcuNHW+jrSkV6wGyNxEhJUulj2AUDJSEr735kA+IeMzGypUmePuebHSSVrxCbP47sNwNdik9fOQDVv3ly2bNlSrNvUoWIBACgqYhMAwCeugQIAAAAAX0ACBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIBNJFAAAAAAYJPXbqQLAADQMaKRx9tMStvs8TYB+A/OQAEAAACATSRQAAAAAGATCRQAAAAA2EQCBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIBNJFAAAAAAYBMJFAAAAAD4egK1bds26d+/v7d3AwAAg7gEAHB0AgUAAAAATuOoBOrMmTNyzz33yK233iqTJ082y+bNmyfNmjWT5s2bS1JSkln2ww8/SNu2baVVq1by8ssvm2XTpk2Tpk2bSrt27eTjjz/Od/tZWVmSkZGRawIAwFtxSRGbAMC3OCqB+uSTT+Taa6+Vr7/+Wm6++WY5e/asxMfHy6pVq2Tp0qXyzDPPmPWefvppSUxMlDVr1pjnfv/9d5k/f7553YoVKyQuLi7f7eu2wsPD3VNUVJSH3yEAwJeUdFxSxCYA8C2OSqB27dolTZo0MfMaqA4fPixXXXWVlC1bVsLCwqR06dKmNzA5OVm6dOkiMTExsn//fklJSZEXX3xRhg0bZurTd+7cme/2R48eLenp6e5JXwcAgLfikiI2AYBvCREH0V6+TZs2Sbdu3Uw5xOWXXy5btmyRkydPyqlTp8wUEhIiDRs2lIULF5qeOu0NDA4ONuvMnDlT1q1bJxMnTpQZM2acs/3Q0FAzAQDghLikiE0A4FsclUDdfffdpra8ffv2UqdOHSlVqpQpi2jdurUJRs8//7xZT3v1unbtKtnZ2SboaG354MGD5bfffjO15C+88IK33woAwA8QlwAAeQVZlmVJgNILdbW3MEbiJCSotLd3B8AFJKVt9vYuoBhlZGZLlTp7TMmalsLh/xCbSh7HEwAXE5scdQ0UAAAAADgZCRQAAAAA2EQCBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIBNJFAAAAAAYFOI3RUBAAD8QceIRl5pNylts1faBVC8OAMFAAAAADaRQAEAAACATSRQAAAAAGATCRQAAAAA2EQCBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIBNJFAAAAAA4E8J1G+//SZLly4189HR0d7eHQBAgCMuAUDg8rkECgAAbyMuAUDg8okE6u2335aPPvpIYmJi5NixY9KvXz9p1KiRzJkzxzy/Z88e6dixo3n+iSeeOO92srKyJCMjI9cEAIC34pIiNgGAb/GJBGrw4MHSs2dPWblypRw6dEhef/11Wb16tbz22mvm+aefflreeust8/zJkyflhx9+yHc78fHxEh4e7p6ioqI8/E4AAP6guOKSIjYBgG/xiQQqp5o1a0pYWJiZzp49a5Zt375dBg4caHr6Nm7cKKmpqfm+dvTo0ZKenu6eUlJSPLz3AAB/czFxSRGbAMC3hIgPKF26tDsoBQUFnfN83bp15eWXX5arr75aLMtyr5tXaGiomQAAcEJcUsQmAPAtPnEGqkGDBvLjjz9Kjx495K+//jrn+YkTJ8rDDz8sbdu2lQ4dOkhaWppX9hMAEBiISwAQuIIs7RoLUHqhrtabx0ichASV9vbuALiApLTN3t4FFKOMzGypUmePKVnT0jf8H2KT/+I4BvhHbPKJM1AAAAAA4AQkUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIBNJFAAAAAAYBMJFAAAAADYRAIFAAAAADaRQAEAAACATSRQAAAAAGBTiN0VAQAAUHQdIxp5vM2ktM0ebxPwd5yBAgAAAACbSKAAAAAAwCYSKAAAAACwiQQKAAAAAGwigQIAAAAAm0igAAAAAMAmEigAAAAAsIkECgAAAABsIoECAAAAAJtIoAAAAADAJhIoAAAAALCJBAoAAAAAnJ5ArVy5UqKioiQhIUGio6NzPed63L9/f+ncufM5y/W1I0aMMPPr16+Xli1byl9//SV9+/aVyMjI87aZlZUlGRkZuSYAAFyITQAAR5+B6tmzpwwaNOiC66SmpkpycnK+z23dulWGDRsmiYmJUrlyZZk9e7ZUq1btvNuKj4+X8PBw96RBEgCAnIhNAACfLuHT3ryXXnrpnOV79+6VAQMGyPz586Vq1aq2tjV69GhJT093TykpKSWwxwAAf0dsAoDA5fgEqkmTJnLkyBHZt29fruXLly+XZs2ayTXXXGN7W6GhoRIWFpZrAgCgsIhNABC4HJFABQUFuedPnjwp5cqVy/X88OHD5ZVXXsm1THv4Dhw4IDNmzPDYfgIAAgexCQDg2ASqRo0asnnzZjO/du1aadCgQa7nO3ToIJs2bZKjR4+6lwUHB8ucOXPk3XfflaVLl3p8nwEA/o3YBADIT4g4wIQJE2Tw4MFy5swZ08OngSevRx55RO69995cy8qXLy8ff/yxxMbGmgt0b7zxRg/uNQDAnxGbAAD5CbIsyxIv2LBhgzz00EMydOjQAkc7skuHit2+fbts3LjR1vo6VKyOeBQjcRISVLpY9gFAyUhK+9+ZAPiHjMxsqVJnjxk0wUnX/BCb4G84dgLFH5u8dgaqefPmsmXLlmLdpg4VCwBAURGbAAA+cQ0UAAAAAPgCEigAAAAAsIkECgAAAABsIoECAAAAAJtIoAAAAADAJhIoAAAAALCJBAoAAAAAbPLafaAAAABQsjpGNPJ4m9y8F/6OM1AAAAAAYBMJFAAAAADYRAIFAAAAADaRQAEAAACATSRQAAAAAGATCRQAAAAA2EQCBQAAAAA2kUABAAAAQEkmUCkpKZKamup+vHHjRnn88cclISGhKJsDAOCiEJcAAI5OoHr37i3ffPONmT906JB06NDBBKtnnnlGxo0bV9z7CADABRGXAACOTqC2bdsmTZs2NfPz58+XG264QdatWydz5syRWbNmFfc+AgBwQcQlAICjE6jTp09LaGiomf/666/lrrvuMvP16tWTgwcPFu8eAgBQAOISAMDRCdT1118v06ZNkzVr1siyZcukU6dOZnlaWppceumlxb2PAABcEHEJAODoBGrixIkyffp0iYmJkV69eknDhg3N8s8++8xdQlFcNmzYIM2aNZO2bdvKs88+K08++aS0adPGtLN582Y5fPiw3H777e7127dvLxkZGfluKysryzyXcwIA+D5PxiVFbAKAwBVSlBdpgDpy5Ig5yFepUsW9fNCgQVK+fPni3D9ZvHixjB07Vm677TbJzs6WkydPmjY2bdokkyZNMvXtZcqUMSUaJ06ckCuuuELCwsLy3VZ8fLw899xzxbp/AADv82RcUsQmAAhcQZZlWYV9kQaNAQMGyNVXXy0lTUdTev755+XPP/+UPn36yPfff2/q21VISIgZdWnRokWyb98+OXbsmDRu3FjuuOOO8/by6eSigTYqKkpiJE5CgkqX+HsBUHRJaZu9vQsoRhmZ2VKlzh5JT08/b2Lh1LikiE3A+XG8hr/HpiIlUI0aNTIjHmm5wsCBA6Vbt27ui3eLm/bclStXTk6dOiVNmjSR8PBwWbt2rfz4448yfPhwWblypXmuc+fO5iLiFStWmOBlhwYp3R5BCnA+ArJ/Ke4EypNxSRGbgPPjeA1/j01FugZK67u1t00v2h02bJhUq1ZNBg8ebJYVN61pb926tSnP6N+/v1xyySVmfsGCBe51tExCR1rSmne7AQoA4D88GZcUsQkAAleRzkDlpD1rn3/+ucycOVOSkpJMsNDePw0o2oPmKY8++qj069dPoqOjbb+GXj7Ad9Cj6V+K+wyUE+OSIjYhEHG8hq8q0TNQOWn+pcFKSxV0Xi/efeONN0z99kcffSSeMGTIEDl69GihAhQAwD85IS4pYhMA+Kci1xRonbf27s2dO9fUmfft21fefPNNufbaa83zr7/+ujz22GPSs2dPKWlvvfVWibcBAHA2J8UlRWwCAP9UpDNQDRo0kObNm8vevXvl3//+t6SkpMiLL77oDlJK78Oh98EAAKCkEZcAAI4+A3XPPfeY4WKvvPLK865z2WWXmXtjAABQ0ohLAADHnoHSuvJZs2Zxp3QAgCMQlwAAjk6gSpcube64DgCAExCXAACOvwZq6NChMnHiRDlz5kzx7xEAAIVEXAIAOPoaKL0x4fLly2Xp0qXmwt0KFSrkej4xMbG49g8AgAIRlwAAjk6gKleuLN26dSv+vQEAoAiISwAARydQep8NAACcgrgEOEfHiEZeaTcpbbNX2kXgKdI1UErrzL/++muZPn26ZGZmmmVpaWny999/F+f+AQBgC3EJAODYM1D79u2TTp06yf79+yUrK0s6dOgglSpVMhfw6uNp06YV/54CAHAexCUAgKPPQA0bNkyio6Plzz//lHLlyrmXd+nSxVzECwCAJxGXAACOPgO1Zs0aWbdunZQpUybX8muuuUYOHDhQXPsGAIAtxCUAgKPPQGVnZ8vZs2fPWZ6ammpKJgAA8CTiEgDA0QlUbGysTJkyxf04KCjIXKQ7duxYue2224pz/wAAKBBxCQDg6BK+V155RTp27Cj169eXkydPSu/evWXnzp1y2WWXydy5c4t/LwEAuADiEgDA0QlUZGSkbNmyRebNmyfJycmml2/gwIHSp0+fXBfvAgDgCcQlAICjEyjzwpAQue+++4p3bwAAKCLiEgDAsQnU7NmzL/h83759i7o/AAAUGnEJAODoBErvt5HT6dOn5fjx42b42PLlyxOoAAAeRVwCADh6FD69UWHOSWvNd+zYIbfccovXLtb97rvvpGXLlnLTTTfJBx984JV9AAB4hxPjkiI2AYD/KVIClZ/atWvLiy++eE4voKdUr15dVqxYYW6kOHXqVK/sAwDAObwdlxSxCQD8T0ixbiwkRNLS0sQbrrrqKvNz9erVUrdu3XzXycrKMpNLRkaGx/YPAOB53oxLitgEAP6nSAnUZ599luuxZVly8OBBeeONN0ypgrccPnxYRo4cKV988UW+z8fHx8tzzz3n8f0CAJQsp8YlRWwCAP8SZGmUKaTg4NyVf3rH98svv1zatWtnbmaoJQveMH/+fDl06JA89thjtnv5oqKiJEbiJCSotAf3FEBhJaVt9vYuoBhlZGZLlTp7JD09XcLCwi56e06NS4rYBHgGcQKeik1FOgOVnZ0tTq13r1Wr1nmfDw0NNRMAwL84NS4pYhMA+JciJVBPPvmk7XVfffVV8ZTff/9dTpw4IU2aNPFYmwAA73NqXFLEJgDwL0VKoDZt2iQ//fSTnDlzxn1R7K+//iqlSpUyQ7XmLKHwpE6dOnm0PQCAMzg1LiliEwD4lyIlUHfeeadUqlRJ3nvvPalSpYpZpvfdeOCBB6RVq1YyfPjw4t5PAADOi7gEAHD0IBJXXnmlLF26VK6//vpcy7dt2yaxsbFeHTK2MPRC3fDwcC7UBXwAFwf7l+IeRMJf4pIiNgFFQ5yAp2JTcFEP7josa166LDMzsyibBACgyIhLAABPKVIC1aVLF1MWkZiYKKmpqWZatGiRDBw4ULp27Vr8ewkAwAUQlwAAjr4Gatq0aTJixAjp3bu3nD59+n8bCgkxgWrSpEnFvY8AAFwQcQkA4OhroFyOHTsmu3fvNvN6j4sKFSqIL6HOHPAd1Lb7l+K+Bspf4pIiNgFFQ5yAo2+k66KB6cYbb7yYTQAAUGyISwAAR14DBQAAAACBiAQKAAAAAGwigQIAAAAAmy7qGigAAADACTpGNPJ4mwxcEZg4AwUAAAAANpFAAQAAAIBNJFAAAAAAYBMJFAAAAADYRAIFAAAAADaRQAEAAACATSRQAAAAAGATCRQAAAAA2EQCBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAAPhCArVy5UqJioqShIQEiYmJkVatWpmfOqWnp0t2drb885//NMtvueUWee2119yvHTFihLRs2dIsHz9+vFn21FNPSeXKleXvv//Ot72srCzJyMjINQEAkBOxCQBwISHiZT179pRBgwbJhx9+KEuWLJGKFSu6n3vnnXfk6NGjsmbNGjlz5ozExcVJ/fr1pXr16rJv3z759ttvzXp//vmn+fnSSy/Jxo0bz9tWfHy8PPfccx54VwAAX0ZsAgD4ZAnfvHnzZOTIkWY+JCREnnzySZk7d66ULVtWdu7cKb/88ot5rkqVKra2N3r0aNN76JpSUlJKdP8BAP6H2AQAgc1RCVTnzp1NiYT+VGlpaRIREeF+PjIy0iyrVauWPP300zJkyBCpU6eOfPrpp7a2HxoaKmFhYbkmAAAuhNgEAHBUCV9OecsktBxCg1KNGjXM49TUVHfQuvfee8106NAhad++vSmhAACguBGbAACOPQOVlwahl19+2cxrnfmrr75qlmnt+X//+1+zXC/MLV26tJf3FAAQKIhNABDYHHUGSssjSpUqZeZnz54tAwcONCMd6WhGlmVJjx49pEOHDrJ3717p16+fWabB65lnnvH2rgMA/BSxCQDgmARKL7hdtmyZGSpWh4093+hEeWnZxOrVq89ZrkPFatlEcLCjT6wBAByM2AQAuJAgS7vKApTeayM8PFxiJE5Cgii1AJwsKW2zt3cBxSgjM1uq1NljRp1j0ITciE2A7yA2BWZsojsMAAAAAGwigQIAAAAAm0igAAAAAMAmEigAAAAAsIkECgAAAABsIoECAAAAAJtIoAAAAADAF26kCwAAAPiqjhGNvNIu95/yLs5AAQAAAIBNJFAAAAAAYBMJFAAAAADYRAIFAAAAADaRQAEAAACATSRQAAAAAGATCRQAAAAA2EQCBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAANhEAgUAAAAAvpBArVy5UqKioiQhIUFiYmKkVatW0rRpUxk/frx7nRYtWsi4cePcj2fNmiW1a9eWdu3amfWnT5/ufi42Nlaio6M9/j4AAP6D2AQAcPQZqJ49e8qgQYPM/JIlS2TdunWycOFCSU1NlZSUFImMjDTBLKdhw4bJihUrJCkpSRITE2Xx4sVm+dKlSy/YVlZWlmRkZOSaAADIi9gEAHBsApVXSEiI1K9fXw4cOGCCVZ8+faRevXqyffv2c9YtX768jBo1ShYtWmRr2/Hx8RIeHu6etIcRAICCEJsAAI5NoI4fPy7JyclSs2ZN02vXqVMn6dWrlyxYsCDf9SMiIuTgwYO2tj169GhJT093T9qLCABAQYhNAACXEHGQzp07S3BwsIwcOdKUNGzbtk3i4uLEsiwTVMaMGXPOa9LS0kygsiM0NNRMAADYRWwCADg2gdI684oVK5r5KVOmyOTJk6V79+7m8ZAhQ2THjh251j9x4oRMmjTJ1J0DAFASiE0AAEeX8Llo7Xjbtm3dj3V+/vz5Zn7q1KlmpCMd2ahr166mlAIAgJJGbAIAePUMVNmyZWXZsmVmqNi8oxmtWbMm1+MePXq45/v375/v9jRo2S2ZAAAgP8QmAIBjE6jmzZvLli1bim17BQ0VCwBAQYhNAACfLOEDAAAAAKchgQIAAAAAm0igAAAAAMAmEigAAAAAsIkECgAAAABsIoECAAAAAJtIoAAAAADAJhIoAAAAAPCFG+kCAAAAKJyOEY083mZS2maPt+lUnIECAAAAAJtIoAAAAADAJhIoAAAAALCJBAoAAAAAbCKBAgAAAACbSKAAAAAAwCYSKAAAAACwiQQKAAAAAGwigQIAAAAAm0igAAAAAMAmEigAAAAA8IUEauXKlRIVFSUJCQkSExMjrVq1kqZNm8r48ePd67Ro0ULGjRvnfjxr1iypXbu2tGvXzqw/ffp093OxsbESHR3t8fcBAPAfxCYAgKPPQPXs2VMGDRpk5pcsWSLr1q2ThQsXSmpqqqSkpEhkZKQJZjkNGzZMVqxYIUlJSZKYmCiLFy82y5cuXXrBtrKysiQjIyPXBABAXsQmAIBjE6i8QkJCpH79+nLgwAETrPr06SP16tWT7du3n7Nu+fLlZdSoUbJo0SJb246Pj5fw8HD3pD2MAAAUhNgEAHBsAnX8+HFJTk6WmjVrml67Tp06Sa9evWTBggX5rh8RESEHDx60te3Ro0dLenq6e9JeRAAACkJsAgC4hIiDdO7cWYKDg2XkyJGmpGHbtm0SFxcnlmWZoDJmzJhzXpOWlmYClR2hoaFmAgDALmITAMCxCZTWmVesWNHMT5kyRSZPnizdu3c3j4cMGSI7duzItf6JEydk0qRJpu4cAICSQGwCADi6hM9Fa8fbtm3rfqzz8+fPN/NTp041Ix3pyEZdu3Y1pRQAAJQ0YhMAwKtnoMqWLSvLli0zQ8XmHc1ozZo1uR736NHDPd+/f/98t6dBy27JBAAA+SE2AQAcm0A1b95ctmzZUmzbK2ioWAAACkJsAgD4ZAkfAAAAADgNCRQAAAAA2EQCBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIAv3EgXAAAAgPN1jGjklXaT0jaL03AGCgAAAABsIoECAAAAAJtIoAAAAADAJhIoAAAAALCJBAoAAAAAbCKBAgAAAACbSKAAAAAAwCYSKAAAAACwiQQKAAAAAGwigQIAAAAAm0igAAAAAMAmEigAAAAA8IUEauXKlRIVFSUJCQkSExMjrVq1kqZNm8r48ePd67Ro0ULGjRvnfjxr1iypXbu2tGvXzqw/ffp093OxsbESHR3t8fcBAPAfxCYAgKPPQPXs2VMGDRpk5pcsWSLr1q2ThQsXSmpqqqSkpEhkZKQJZjkNGzZMVqxYIUlJSZKYmCiLFy82y5cuXXrBtrKysiQjIyPXBABAXsQmAIBjE6i8QkJCpH79+nLgwAETrPr06SP16tWT7du3n7Nu+fLlZdSoUbJo0SJb246Pj5fw8HD3pD2MAAAUhNgEAHBsAnX8+HFJTk6WmjVrml67Tp06Sa9evWTBggX5rh8RESEHDx60te3Ro0dLenq6e9JeRAAACkJsAgC4hIiDdO7cWYKDg2XkyJGmpGHbtm0SFxcnlmWZoDJmzJhzXpOWlmYClR2hoaFmAgDALmITAMCxCZTWmVesWNHMT5kyRSZPnizdu3c3j4cMGSI7duzItf6JEydk0qRJpu4cAICSQGwCADi6hM9Fa8fbtm3rfqzz8+fPN/NTp041Ix3pyEZdu3Y1pRQAAJQ0YhMAwKtnoMqWLSvLli0zQ8XmHc1ozZo1uR736NHDPd+/f/98t6dBy27JBAAA+SE2AQAcm0A1b95ctmzZUmzbK2ioWAAACkJsAgD4ZAkfAAAAADgNCRQAAAAA2EQCBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIAv3EgXAAAAAM6nY0Qj8ZQz1mkR2VPgepyBAgAAAACbSKAAAAAAwCYSKAAAAACwiQQKAAAAAGwigQIAAAAAm0igAAAAAMAmEigAAAAAsIkECgAAAABsIoECAAAAAJtIoAAAAADAJhIoAAAAAPCFBGrlypUSFRUlCQkJEhMTI61atZKmTZvK+PHj3eu0aNFCxo0b5348a9YsqV27trRr186sP336dPdzsbGxEh0d7fH3AQDwH8QmAICjz0D17NlTBg0aZOaXLFki69atk4ULF0pqaqqkpKRIZGSkCWY5DRs2TFasWCFJSUmSmJgoixcvNsuXLl16wbaysrIkIyMj1wQAQF7EJgCAYxOovEJCQqR+/fpy4MABE6z69Okj9erVk+3bt5+zbvny5WXUqFGyaNEiW9uOj4+X8PBw96Q9jAAAFITYBABwbAJ1/PhxSU5Olpo1a5peu06dOkmvXr1kwYIF+a4fEREhBw8etLXt0aNHS3p6unvSXkQAAApCbAIAuISIg3Tu3FmCg4Nl5MiRpqRh27ZtEhcXJ5ZlmaAyZsyYc16TlpZmApUdoaGhZgIAwC5iEwDAsQmU1plXrFjRzE+ZMkUmT54s3bt3N4+HDBkiO3bsyLX+iRMnZNKkSabuHACAkkBsAgA4uoTPRWvH27Zt636s8/PnzzfzU6dONSMd6chGXbt2NaUUAACUNGITAMCrZ6DKli0ry5YtM0PF5h3NaM2aNbke9+jRwz3fv3//fLenQctuyQQAAPkhNgEAHJtANW/eXLZs2VJs2ytoqFgAAApCbAIA+GQJHwAAAAA4DQkUAAAAANhEAgUAAAAANpFAAQAAAIBNJFAAAAAAYBMJFAAAAADYRAIFAAAAAL5wHyhvsyzL/Dwjp0X+NwvAoTIys729CyhGGX9n5zoO4/8QmwDAO8xx10ZsCugEKjMz0/xcK196e1cAFKBKHW/vAUrqOBweHu7t3XAUYhMAODs2BVkB3P2XnZ0taWlpUqlSJQkKCirUazMyMiQqKkpSUlIkLCysxPYxENv0VruB0qa32g2UNr3Vrq+1qaFHA1RERIQEB1NNXhyxid93/2vTW+0GSpveapf36vuxKaDPQOkHExkZeVHb0P8YT/7yB1Kb3mo3UNr0VruB0qa32vWlNjnzVDKxid93/2vTW+0GSpveapf36ruxiW4/AAAAALCJBAoAAAAAbCKBKqLQ0FAZO3as+Umb/tFuoLTprXYDpU1vtRsobeL8+H33vza91W6gtOmtdnmvvt9mQA8iAQAAAACFwRkoAAAAALCJBAoAAAAAbCKBAgAAAACbSKAAAAAAwCYSKMABYmJi5PHHH/f2bgAAYBCXgPMjgQIAAAAAm0igAAAAAMAmEijAgRYvXizh4eEyZ84cSUlJkXvuuUcqV64sl1xyicTFxclvv/1m1lu9erWULl1aDh06lOv1WnbRqlUrM79v3z658847pUqVKlKhQgW5/vrr5csvv/TK+wIA+CbiEvB/SKAAh/nwww+lV69eJkhpgOrYsaNUqlRJ1qxZI99++61UrFhROnXqJKdOnZLWrVtLzZo15f3333e//vTp0+a1AwYMMI+HDh0qWVlZJqht3bpVJk6caLYBAIAdxCUgt5A8jwF40ZtvvinPPPOMfP7559KmTRv54IMPJDs7W959910JCgoy68ycOdP0+q1cuVJiY2Nl4MCBZtnIkSPN8/rakydPmiCn9u/fL926dZMGDRqYxxrYAACwg7gEnIszUIBDLFy4UJ544glZtmyZCVJqy5YtsmvXLtPTp71zOmm5hAai3bt3m3X69+9v1tmwYYN5PGvWLBOktCxCPfbYY/L8889Ly5YtZezYsZKcnOzFdwkA8BXEJSB/JFCAQzRu3Fguv/xymTFjhliWZZb9/fff0qRJE9m8eXOu6ddff5XevXubda644gpTS669fb///rssWbLEXSah/vGPf8iePXvk/vvvN6US0dHR8vrrr3vtfQIAfANxCcgfCRTgELVq1ZJvvvlGPv30U3n00UfNsptuukl27txpgtG1116ba9KLeXMGo48++kgSEhLMdrRXL6eoqCh5+OGHJTExUYYPHy7vvPOOx98fAMC3EJeA/JFAAQ5Sp04dE6wWLVpkRizq06ePXHbZZWaEI71Yd+/evabGXMsfUlNT3a/TC3rDwsJMScQDDzyQa5u6naSkJPPan376yWz/uuuu88K7AwD4GuIScC4GkQAcpm7durJixQpzF/hSpUqZUYpGjRolXbt2lczMTLnyyiulffv2JjC5BAcHm5rzCRMmSN++fXNt7+zZs2bEIw1s+hodKWny5MleeGcAAF9EXAJyC7JcRa0AfJqOenT48GH57LPPvL0rAAAQl+C3OAMF+Lj09HRzEa7ep4MgBQDwNuIS/B0JFODjtA5948aN5mLcDh06eHt3AAABjrgEf0cJHwAAAADYxCh8AAAAAGATCRQAAAAA2EQCBQAAAAA2kUABAAAAgE0kUAAAAABgEwkUAAAAANhEAgUAAAAANpFAAQAAAIDY8/8BmeunwqnWL88AAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n"
     ]
    }
   ],
   "execution_count": 369
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Transformer Model",
   "id": "e61a235d885f7c3c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.316371Z",
     "start_time": "2025-03-24T10:58:41.303822Z"
    }
   },
   "cell_type": "code",
   "source": [
    "@dataclass\n",
    "class TransformerOutput:\n",
    "    \"\"\"\n",
    "    Transformer模型的完整输出容器\n",
    "    \n",
    "    属性说明：\n",
    "    logits: Tensor\n",
    "        解码器输出的原始预测分数（未经过Softmax）\n",
    "        形状：[batch_size, trg_len, vocab_size]\n",
    "        用于计算交叉熵损失或生成最终预测\n",
    "    encoder_last_hidden_states: Tensor\n",
    "        编码器最后一层的隐藏状态\n",
    "        形状：[batch_size, src_len, hidden_dim]\n",
    "        包含源语言的高层语义表示\n",
    "    encoder_attn_scores: List[Tensor]\n",
    "        编码器各层的自注意力分数列表\n",
    "        每个元素形状：[batch_size, num_heads, src_len, src_len]\n",
    "        用于可视化编码器内部注意力模式\n",
    "    decoder_last_hidden_states: Tensor\n",
    "        解码器最后一层的隐藏状态\n",
    "        形状：[batch_size, trg_len, hidden_dim]\n",
    "        包含目标语言的生成状态表示\n",
    "    decode_self_attn_scores: List[Tensor]\n",
    "        解码器各层的自注意力分数列表\n",
    "        每个元素形状：[batch_size, num_heads, trg_len, trg_len]\n",
    "        用于分析目标序列生成时的注意力模式\n",
    "    preds: Optional[Tensor] = None\n",
    "        推理阶段生成的预测token索引\n",
    "        形状：[batch_size, trg_len]\n",
    "        实际应用时用于获取最终结果\n",
    "    \"\"\"\n",
    "    logits: Tensor\n",
    "    encoder_last_hidden_states: Tensor\n",
    "    encoder_attn_scores: List[Tensor]  #画图\n",
    "    decoder_last_hidden_states: Tensor\n",
    "    decoder_self_attn_scores: List[Tensor]  #画图\n",
    "    decoder_cross_attn_scores: List[Tensor]  #画图\n",
    "    preds: Optional[Tensor] = None\n",
    "\n",
    "\n",
    "class TransformerModel(nn.Module):\n",
    "    \"\"\"完整的Transformer模型（编码器-解码器架构）\"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        \"\"\"\n",
    "        参数说明：\n",
    "        config: dict\n",
    "            模型配置字典，包含关键参数：\n",
    "            - d_model: 隐藏层维度（如512）\n",
    "            - num_encoder_layers: 编码器层数（如6）\n",
    "            - num_decoder_layers: 解码器层数（如6）\n",
    "            - pad_idx: 填充token的索引\n",
    "            - bos_idx: 句子起始符索引\n",
    "            - eos_idx: 句子结束符索引\n",
    "            - vocab_size: 词表大小\n",
    "            - dropout: dropout概率\n",
    "            - max_length: 最大序列长度\n",
    "            - share_embedding: 是否共享编码器/解码器嵌入矩阵\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        # ===== 超参数初始化 =====\n",
    "        self.hidden_size = config[\"d_model\"]  # 特征维度\n",
    "        self.num_encoder_layers = config[\"num_encoder_layers\"]  # 编码器层数\n",
    "        self.num_decoder_layers = config[\"num_decoder_layers\"]  # 解码器层数\n",
    "        self.pad_idx = config[\"pad_idx\"]  # 填充符索引（用于掩码计算）\n",
    "        self.bos_idx = config[\"bos_idx\"]  # 句子起始符（Begin of Sequence）\n",
    "        self.eos_idx = config[\"eos_idx\"]  # 句子结束符（End of Sequence）\n",
    "        self.vocab_size = config[\"vocab_size\"]  # 词表大小\n",
    "        self.dropout_rate = config[\"dropout\"]  # Dropout概率\n",
    "        self.max_length = config[\"max_length\"]  # 最大序列长度（用于推理）\n",
    "        self.share = config[\"share_embedding\"]  # 是否共享词嵌入权重\n",
    "\n",
    "        # ===== 网络层初始化 =====\n",
    "        # 源语言嵌入层（编码器输入）\n",
    "        self.src_embedding = TransformerEmbedding(config)\n",
    "\n",
    "        # 目标语言嵌入层（解码器输入）\n",
    "        if self.share:\n",
    "            # 共享词嵌入矩阵：编码器/解码器使用相同的嵌入参数\n",
    "            self.trg_embedding = self.src_embedding\n",
    "            # 输出投影层：直接使用嵌入矩阵的转置（Tied Embeddings）\n",
    "            self.linear = lambda x: torch.matmul(\n",
    "                x, self.trg_embedding.get_word_embedding_weights().T\n",
    "            )\n",
    "            # 原理：减少参数数量，提升训练稳定性（参考：Press & Wolf 2016）\n",
    "        else:\n",
    "            # 独立的目标语言嵌入层\n",
    "            self.trg_embedding = TransformerEmbedding(config)\n",
    "            # 独立的输出投影层\n",
    "            self.linear = nn.Linear(self.hidden_size, self.vocab_size)\n",
    "\n",
    "        # 编码器模块\n",
    "        self.encoder = TransformerEncoder(config)\n",
    "\n",
    "        # 解码器模块\n",
    "        self.decoder = TransformerDecoder(config)\n",
    "\n",
    "        # 参数初始化\n",
    "        self._init_weights()\n",
    "\n",
    "    def _init_weights(self):\n",
    "        \"\"\"使用Xavier均匀分布初始化权重\"\"\"\n",
    "        for p in self.parameters():\n",
    "            if p.dim() > 1:  # 只初始化维度大于1的参数（跳过bias等）\n",
    "                nn.init.xavier_uniform_(p)\n",
    "        # 原理：保持各层激活值的方差稳定，加速收敛\n",
    "\n",
    "    def generate_square_subsequent_mask(self, sz: int) -> Tensor:\n",
    "        \"\"\"\n",
    "        生成因果掩码（Causal Mask）\n",
    "        参数：\n",
    "            sz: 目标序列长度\n",
    "        返回：\n",
    "            mask: [sz, sz]的布尔张量，上三角部分为True（需要屏蔽）\n",
    "        \"\"\"\n",
    "        mask = (torch.triu(torch.ones(sz, sz)) == 0).transpose(-1, -2).bool()  # 生成上三角为False的掩码\n",
    "        return mask  # 转换为下三角掩码\n",
    "        # 示例：当sz=3时，掩码为：\n",
    "        # [[0, 1, 1],\n",
    "        #  [0, 0, 1],\n",
    "        #  [0, 0, 0]]\n",
    "\n",
    "    def forward(\n",
    "            self, encoder_inputs, decoder_inputs, encoder_inputs_mask=None\n",
    "    ) -> TransformerOutput:\n",
    "        \"\"\"\n",
    "        前向传播（训练阶段）\n",
    "        \n",
    "        参数说明：\n",
    "        encoder_inputs: [batch_size, src_len]\n",
    "            源语言token索引序列\n",
    "        decoder_inputs: [batch_size, trg_len]\n",
    "            目标语言token索引序列\n",
    "        encoder_inputs_mask: [batch_size, src_len]（可选）\n",
    "            源语言序列的填充掩码（True表示需要屏蔽的位置）\n",
    "        decoder_inputs_mask: [batch_size, trg_len]（可选）\n",
    "            目标语言序列的填充掩码\n",
    "        \n",
    "        处理流程：\n",
    "        1. 生成编码器/解码器所需的注意力掩码\n",
    "        2. 通过编码器处理源语言输入\n",
    "        3. 通过解码器生成目标语言输出\n",
    "        4. 返回所有中间结果\n",
    "        \"\"\"\n",
    "        # ===== 掩码生成 =====\n",
    "        # 编码器掩码处理\n",
    "        if encoder_inputs_mask is None:\n",
    "            # 自动生成源语言填充掩码（pad的位置为True）\n",
    "            encoder_inputs_mask = encoder_inputs.eq(self.pad_idx)\n",
    "        # 调整掩码维度：[batch, 1, 1, src_len]（适配多头注意力）\n",
    "        encoder_inputs_mask = encoder_inputs_mask.unsqueeze(1).unsqueeze(2)\n",
    "\n",
    "        # 解码器自注意力掩码（因果掩码 + 填充掩码）\n",
    "        look_ahead_mask = self.generate_square_subsequent_mask(decoder_inputs.shape[1])\n",
    "        look_ahead_mask = (\n",
    "            look_ahead_mask.unsqueeze(0).unsqueeze(0).to(decoder_inputs.device)\n",
    "        )\n",
    "\n",
    "        decoder_inputs_mask = decoder_inputs.eq(self.pad_idx)  # [batch_size, trg_len]，和上面encoder_inputs_mask一致\n",
    "        # print(decoder_inputs_mask.shape)\n",
    "        decoder_inputs_mask = decoder_inputs_mask.unsqueeze(1).unsqueeze(2)  # [batch_size, 1, 1, trg_len]\n",
    "        # print(decoder_inputs_mask.shape)\n",
    "        decoder_inputs_mask = decoder_inputs_mask + look_ahead_mask  # [batch_size, 1, 1, trg_len]与[1, 1, trg_len, trg_len]相加，得到decoder的自注意力mask\n",
    "\n",
    "        # ===== 编码器前向传播 =====\n",
    "        encoder_inputs_embeds = self.src_embedding(encoder_inputs)\n",
    "        encoder_outputs = self.encoder(encoder_inputs_embeds, encoder_inputs_mask)\n",
    "\n",
    "        # ===== 解码器前向传播 =====\n",
    "        decoder_inputs_embeds = self.trg_embedding(decoder_inputs)\n",
    "        decoder_outputs = self.decoder(\n",
    "            decoder_inputs_embeds=decoder_inputs_embeds,\n",
    "            encoder_outputs=encoder_outputs.last_hidden_states,\n",
    "            attn_mask=decoder_inputs_mask,  #用于decoder的自注意力,广播去做计算\n",
    "            cross_attn_mask=encoder_inputs_mask,  #用于decoder的交叉注意力,广播去做计算\n",
    "        )\n",
    "\n",
    "        # ===== 输出投影 =====\n",
    "        logits = self.linear(decoder_outputs.last_hidden_states)\n",
    "\n",
    "        return TransformerOutput(\n",
    "            logits=logits,\n",
    "            encoder_last_hidden_states=encoder_outputs.last_hidden_states,\n",
    "            encoder_attn_scores=encoder_outputs.attn_scores,\n",
    "            decoder_last_hidden_states=decoder_outputs.last_hidden_states,\n",
    "            decoder_self_attn_scores=decoder_outputs.self_attn_scores,\n",
    "            decoder_cross_attn_scores=decoder_outputs.cross_attn_scores,\n",
    "        )\n",
    "\n",
    "    @torch.no_grad()\n",
    "    def infer(self, encoder_inputs, encoder_inputs_mask=None) -> Tensor:\n",
    "        \"\"\"\n",
    "        推理方法（自回归生成）\n",
    "        \n",
    "        参数说明：\n",
    "        encoder_inputs: [batch_size, src_len]\n",
    "            源语言token索引序列\n",
    "        encoder_inputs_mask: [batch_size, src_len]（可选）\n",
    "            源语言填充掩码\n",
    "        \n",
    "        处理流程：\n",
    "        1. 初始化以<bos>开头的解码器输入\n",
    "        2. 逐步生成token直到达到max_length或所有序列生成<eos>\n",
    "        3. 返回最终生成的序列\n",
    "        \"\"\"\n",
    "        # 编码器掩码处理\n",
    "        if encoder_inputs_mask is None:\n",
    "            encoder_inputs_mask = encoder_inputs.eq(self.pad_idx)\n",
    "        encoder_inputs_mask = encoder_inputs_mask.unsqueeze(1).unsqueeze(2)\n",
    "\n",
    "        # 生成因果掩码（全长度）\n",
    "        look_ahead_mask = self.generate_square_subsequent_mask(self.max_length)\n",
    "        look_ahead_mask = (\n",
    "            look_ahead_mask.unsqueeze(0).unsqueeze(0).to(encoder_inputs.device)\n",
    "        )\n",
    "\n",
    "        # ===== 编码器处理 =====\n",
    "        encoder_inputs_embeds = self.src_embedding(encoder_inputs)\n",
    "        encoder_outputs = self.encoder(encoder_inputs_embeds)  #因为只支持单样本预测，没有paddings，所以不需要mask\n",
    "\n",
    "        # ===== 自回归生成 =====\n",
    "        # 初始化解码器输入为<bos>\n",
    "        decoder_inputs = torch.Tensor([self.bos_idx] * encoder_inputs.shape[0]).reshape(-1, 1).long().to(\n",
    "            device=encoder_inputs.device)\n",
    "\n",
    "        for cur_len in tqdm(range(1, self.max_length + 1)):\n",
    "\n",
    "            # 解码器前向传播\n",
    "            decoder_inputs_embeds = self.trg_embedding(decoder_inputs)\n",
    "            decoder_outputs = self.decoder(\n",
    "                decoder_inputs_embeds=decoder_inputs_embeds,\n",
    "                encoder_outputs=encoder_outputs.last_hidden_states,\n",
    "                attn_mask=look_ahead_mask[:, :, :cur_len, :cur_len],  #decoder的自注意力mask\n",
    "            )\n",
    "\n",
    "            # 预测下一个token\n",
    "            logits = self.linear(decoder_outputs.last_hidden_states)\n",
    "            next_token = logits.argmax(dim=-1)[:, -1:]  # 取最后一个位置的预测\n",
    "\n",
    "            # 拼接新生成的token\n",
    "            decoder_inputs = torch.cat([decoder_inputs, next_token], dim=-1)\n",
    "\n",
    "            # 提前终止条件：所有序列都生成<eos>\n",
    "            if all((decoder_inputs == self.eos_idx).sum(dim=-1) > 0):\n",
    "                break\n",
    "\n",
    "        return TransformerOutput(\n",
    "            preds=decoder_inputs[:, 1:],\n",
    "            logits=logits,\n",
    "            encoder_last_hidden_states=encoder_outputs.last_hidden_states,\n",
    "            encoder_attn_scores=encoder_outputs.attn_scores,\n",
    "            decoder_last_hidden_states=decoder_outputs.last_hidden_states,\n",
    "            decoder_self_attn_scores=decoder_outputs.self_attn_scores,\n",
    "            decoder_cross_attn_scores=decoder_outputs.cross_attn_scores,\n",
    "        )"
   ],
   "id": "ba06c1258bf64bd0",
   "outputs": [],
   "execution_count": 370
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# 训练",
   "id": "3fc19b2d656f7485"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 损失函数",
   "id": "7a99271aa64d7161"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.321517Z",
     "start_time": "2025-03-24T10:58:41.317445Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class CrossEntropyWithPadding:\n",
    "    \"\"\"\n",
    "    带填充忽略和标签平滑的交叉熵损失计算器\n",
    "    \n",
    "    功能：\n",
    "    1. 自动处理序列任务中的填充位置（padding tokens），在计算损失时忽略这些位置\n",
    "    2. 支持标签平滑（Label Smoothing）技术，提升模型泛化能力\n",
    "    3. 兼容不同长度的序列输入\n",
    "    \n",
    "    参数说明：\n",
    "    config: dict\n",
    "        必须包含键 \"label_smoothing\"（标签平滑系数，取值范围0~1，常用值0.1）\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        # 从配置中获取标签平滑系数（默认0表示不使用）\n",
    "        self.label_smoothing = config[\"label_smoothing\"]\n",
    "\n",
    "    def __call__(self, logits, labels, padding_mask=None):\n",
    "        # logits.shape = [batch size, sequence length, num of classes]\n",
    "        # labels.shape = [batch size, sequence length]\n",
    "        # padding_mask.shape = [batch size, sequence length]\n",
    "        bs, seq_len, nc = logits.shape\n",
    "        loss = F.cross_entropy(logits.reshape(bs * seq_len, nc), labels.reshape(-1), reduce=False,\n",
    "                               label_smoothing=self.label_smoothing)  #label_smoothing表示随机将一个类别的概率设置为0.1，使得模型更加关注其他类别\n",
    "        if padding_mask is None:\n",
    "            loss = loss.mean()\n",
    "        else:\n",
    "            padding_mask = 1 - padding_mask.reshape(-1)  #将padding_mask reshape成一维张量，mask部分为0，非mask部分为1\n",
    "            loss = torch.mul(loss, padding_mask).sum() / padding_mask.sum()\n",
    "        return loss"
   ],
   "id": "78c81a231da62be1",
   "outputs": [],
   "execution_count": 371
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 学习率衰减",
   "id": "6682d24dadd882c"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "学习率曲线特性说明\n",
    "## 预热阶段（0-4000步）\n",
    "\n",
    "    学习率线性增长，帮助模型在训练初期稳定参数\n",
    "\n",
    "    避免大学习率导致的不稳定更新\n",
    "\n",
    "    公式中由arg2项主导\n",
    "\n",
    "## 衰减阶段（4000+步）\n",
    "\n",
    "    学习率按反平方根衰减\n",
    "\n",
    "    符合深度学习后期需要小学习率精细调参的特性\n",
    "\n",
    "    由arg1项主导\n",
    "\n",
    "## 维度缩放因子\n",
    "\n",
    "    d_model^(-0.5) 用于标准化不同维度模型的学习率\n",
    "\n",
    "    防止模型维度增大时学习率过大（维度越大，因子越小）\n",
    "\n",
    "## 设计优势\n",
    "\n",
    "    自动调整：无需手动设置学习率，根据训练进度自动调节\n",
    "    \n",
    "    稳定训练：预热阶段避免早期梯度爆炸\n",
    "    \n",
    "    收敛保障：衰减阶段保证后期参数优化稳定性\n",
    "    \n",
    "    模型适配：通过d_model自动适应不同规模的模型"
   ],
   "id": "efaaa6ccaf57dea8"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.390748Z",
     "start_time": "2025-03-24T10:58:41.322616Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# NoamDecayScheduler 是实现Transformer论文中提出的学习率预热调度器\n",
    "# 数学公式: lr = d_model^(-0.5) * min(step_num^(-0.5), step_num * warmup_steps^(-1.5))\n",
    "class NoamDecayScheduler:\n",
    "    def __init__(self, config):\n",
    "        \"\"\"\n",
    "        初始化Noam学习率调度器\n",
    "        \n",
    "        参数说明：\n",
    "        config : dict\n",
    "            必须包含两个关键参数：\n",
    "            - d_model : 模型维度（论文中的d_model）\n",
    "            - warmup_steps : 预热步数（学习率增长阶段持续时间）\n",
    "        \n",
    "        原理：\n",
    "        1. 训练初期(warmup阶段)：学习率线性增长，帮助模型稳定参数\n",
    "        2. 超过warmup阶段后：学习率按反平方根衰减\n",
    "        3. 通过d_model维度调整学习率幅度，适应不同模型规模\n",
    "        \"\"\"\n",
    "        self.d_model = config[\"d_model\"]  # 模型维度（如512）\n",
    "        self.warmup_steps = config[\"warmup_steps\"]  # 预热步数（如4000）\n",
    "\n",
    "    def __call__(self, step):\n",
    "        \"\"\"\n",
    "        计算给定训练步数的学习率\n",
    "        \n",
    "        参数：\n",
    "        step : int/float/ndarray\n",
    "            当前训练步数（允许标量或数组输入）\n",
    "        \n",
    "        返回：\n",
    "        learning_rate : float/ndarray\n",
    "            计算得到的学习率值\n",
    "            \n",
    "        公式分解：\n",
    "        arg1 = step^(-0.5)  -> 衰减阶段主导项\n",
    "        arg2 = step * warmup_steps^(-1.5) -> 预热阶段主导项\n",
    "        min(arg1, arg2) -> 取两者较小值控制学习率变化曲线\n",
    "        d_model^(-0.5) -> 维度缩放因子\n",
    "        \"\"\"\n",
    "        step += 1  # 避免step=0时出现除零错误，符合论文实现\n",
    "\n",
    "        # 计算衰减阶段的学习率分量（step越大值越小）\n",
    "        arg1 = step ** (-0.5)\n",
    "\n",
    "        # 计算预热阶段的学习率分量（step越大值越大）\n",
    "        arg2 = step * (self.warmup_steps ** (-1.5))\n",
    "\n",
    "        # 模型维度缩放因子（标准化不同维度的学习率幅度）\n",
    "        arg3 = self.d_model ** (-0.5)\n",
    "        # 取两个分量的最小值，确保学习率曲线先升后降\n",
    "        return arg3 * np.minimum(arg1, arg2)\n",
    "\n",
    "\n",
    "# 实例化学习率调度器（参数与Transformer基础模型一致）\n",
    "temp_learning_rate_schedule = NoamDecayScheduler({\"d_model\": 512, \"warmup_steps\": 4000})\n",
    "\n",
    "# 可视化学习率变化曲线（从0到40,000训练步）\n",
    "plt.plot(\n",
    "    temp_learning_rate_schedule(np.arange(0, 40000)),  # 生成0-39999步的学习率\n",
    "    linewidth=2\n",
    ")\n",
    "plt.ylabel(\"Learning Rate\", fontsize=12)  # Y轴标签\n",
    "plt.xlabel(\"Training Step\", fontsize=12)  # X轴标签\n",
    "plt.title(\"Noam Learning Rate Schedule\\n(d_model=512, warmup_steps=4000)\", fontsize=14)\n",
    "plt.grid(alpha=0.3)  # 添加半透明网格线\n",
    "plt.show()"
   ],
   "id": "c153ba71bc07439f",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlUAAAHiCAYAAADBITniAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAkxNJREFUeJztnQd4FOX2xk8KKQSSkARIAqFJlyZVugoKiiiIUsQron9ABAVRUbgIonhR0HsVRbFcRa80saAioAgqSq8ivSeUEAiphPTM/3m/MMvuZhNC2M3uzr6/55ns7sy3s9+Zmd15c875zuelaZomhBBCCCHkuvC+vrcTQgghhBBAUUUIIYQQYgcoqgghhBBC7ABFFSGEEEKIHaCoIoQQQgixAxRVhBBCCCF2gKKKEEIIIcQOUFQRQgghhNgBiipCCCGEEDtAUUUIMSS//fabeHl5yUsvveTsrpBrBOftlltuccpnP/LII+rzT5w44bY2EOdBUUU8Hvx44gcQS69evWy22bRpk9qOH1yjod9EYCNxLHXq1DFda1h8fHwkPDxcevToIUuXLrXLZ5TXzTwvL0/effdd6dixo4SEhIifn59ERUVJhw4d5Omnn5adO3c6vA+EuBq+zu4AIa7Ezz//LGvXrpXbbrvN2V0h10n79u1l//79EhERIa4EhNSUKVPU89zcXDly5Ih8++236rr717/+JZMmTRJXJz8/X+6880755ZdfJDo6Wh544AGpXr26pKSkyI4dO2TOnDkSFBQkN910k7O7Ski5QlFFiJkXIS4uTp5//nnZsmWL+o+fuC8VK1aUxo0bi6vh6+tbJCS5fv166datm7zyyisybtw41XdXZuHChUpQ9e7dW77//nupUKGCxfazZ8/KmTNnnNY/QpwFw3+EXKZRo0byj3/8Q7Zt2yZffvllqd8XGxsrjz32mNSoUUOFQGrWrKleQ6BZs337dhk7dqw0a9ZMhUwCAwOlefPm8tprrymvhS2hhyU1NVVGjx6twivwAOAGDI8AwM3roYcekmrVqqn93XHHHXL48GFxFOvWrZO+ffsqD5C/v780aNBAeV4uXbpk0S4nJ0feeecdFVKNiYlRbdHH++67z2ZoaP78+UrI4vGHH36Qzp07S+XKlZX95mHK48ePK08IBBP2Wbt2bZk+fboUFBSUKqdKP6YXL15UAgaeFuynRYsW8tVXXxUbIh40aJCEhYVJpUqVpHv37uo4YN/4DHzW9QBbYU9mZqbs27fPYtuvv/4qjz76qLo+8dlY2rZtKx9++KFNe8Hvv/9uEWbEMTXnu+++UyHHKlWqSEBAgLoe33jjDeWBKg0bN25Uj6NGjSoiqEBkZKS0bt26yPr09HR1rnCsIRzxHYA368UXX7R5/SckJMiwYcPUtYZr++abby72WGPf06ZNkxtvvFG1DQ0NVdfen3/+abP93r175e6771bXGPpx1113yZ49e2y2Lek8m1+3pQHfi3//+9/q+OC7jM/v2rWrEqfE/aGnihAzXn75ZVm8eLESCbj527phmHPo0CHp0qWLnD9/XgkN/KDjh/mTTz5RwgA/6A0bNjS1/+ijj9R6iCL8iEOI4IcaIZ+tW7fK119/bfNH+Pbbb5esrCx1Y8eNBqKvZ8+esmHDBnXjgNiCsEIoCfvv06ePCn0h1GRP3n//fRkzZoy6YcFeiCSI0FdffVXd/LFAWIKkpCQZP368umHAVtzAjx07pm4eK1euVKKkXbt2RT4DuUUIw+KG98QTT0haWprF9ueee06JBmyH7cuWLVM3PRwn9KM04AYO8ZmcnCwDBgxQ5wHnfeDAgbJq1Sq1Tef06dPSqVMniY+PV54ZiICDBw+qc+KIMDE8Wea8/vrr6rxCUPTv31+F2NBHCBr0480331TtIBQhKiBaIDTN8/9atWpleo5rDSIe/wTgGoeg+OOPP9Rx3bx5c6lyu5AHpl//peXcuXNKjB44cED1B/8kQAjjNWx85pln1HWlAzvx3UL/8M8O3r9kyRJ1zvHPCYSgDq41fKcglCBQH3/8cXXdQDzeeuutyqZ+/fqZ2uM7inYQ1jgG+McA3mmsa9mypTiK7OxsdQ3hO49jgH++cC3++OOPcu+996p/QvBPF3FjNEI8nOPHj2v4KvTq1Uu9fvbZZ9Xrd955x9Rm48aNat2wYcMs3nvrrbeq9R988IHF+rlz56r1t912m8X62NhYLS8vz2JdQUGB9uijj6r2f/75p8W22rVrq/UPPPCAlpuba1r/+uuvq/WhoaHa008/rfahM3r0aLXt66+/LpX9sAntYWNJ7N27V/P19dVatmypJSYmWmybOXOm2scbb7xhWpeVlaWdOnWqyH727NmjVapUSevZs6fF+k8//VTtw9vbW1u9enWx/axbt6525swZ0/rz58+r41C5cmUtOzvbtP7XX39V7adNm2bzmN57770W7X/55ReL60DnoYceUutfffVVi/X//e9/1Xos+KzSgM/29/cvsh7nHXaHh4drmZmZFtuOHTtWpD2uhdtvv13z8fFR15Q56E/37t1tfv7PP/9ssvHixYum9bh+Hn/8cbXtq6++uqod27dvV9eCn5+fNmrUKO3777+3OCe2GDBggNr/5MmTi2w7e/asxfWtH9cnnnhCy8/PN63/+OOP1Xp8pjkPPvigWv/RRx9ZrE9ISNBiYmK0qlWrWhxXHB+0/+KLLyzaT5o0yfTZ+F3QwTVU3HnWr1s8Xu08wHasf/HFFy2+s2lpaVrbtm3V8Tx9+nQxR5C4AxRVxOOxFlVJSUnqJl2tWjUtPT29WFGFmxnWNW3a1OIHEuBG0LhxY7U9Li6uVDcptH3ppZdsCgDrGyf2ifUQJxkZGRbb1q1bp7ZNnTrVrqLqqaeeUu2wf2tgL25cbdq0KdVn9u3bV91AcnJyityc+vfvX2I/P/nkk2K37d69u9SiypZYwbawsDALYQgRhGsBz83BOW/UqNE1iyoIIfQJC26yAwcO1CpUqKBEypIlS7TSAtGMz54/f36pRdU999xj83oCKSkpmpeXlxI/pWHBggVaRESESYRgqVmzpvbII49o27Zts2gbHx+v9n3DDTdYnPPiwL6CgoJM3z8dCC8cp9atW1uIahxT639gdObMmaP298MPP1h8b1u0aFGkLT4P331HiCp8R6pUqaKOgfXvBYAwtf5njrgfDP8RYgXCVC+88IJakGdSXJ2jXbt2qUeENKyT2r29vVU4AqENtENOEUCICsPQEWrCNoQfCn9/C7GV3Iv+1KpVy2Idwn0AYQvrpGZ9m70ThfWSCz/99JOsWbOmyHaESmGTObB91qxZKgyK5GXrvJnExERTf81H7ZVEmzZtiqxDHpseMioNCDPVrVvX5n70fCGA8BpCNshhQt6VOTjnCAuizbWAvCWE6KxDftYhKvNcIVyHCHMePXpUMjIyLLZfy3nGOUQeD8LTtkAukvU5LI4HH3xQhc5Wr16tzi9CcghHI7fo888/l7lz56owHECIGNc5QnFXC6nrIGyO/DHr46SPMtRB2BzHFOfJ1ndVzy+EXQgZ//XXX+o1QovW4PMQlrveHDlb4DpBuBk5fNbnHyCFQO8ncV8oqgixwVNPPaXED/JVkNdjCz3XBz/yttDFgnlO0P33369ynnDDQH4UcpJwk8FN4u2331Y3BmuCg4OLzbspaZutxN/rAXkroLR5S7jB6jlHyFGCAMRNC2IEAgE3N1v2Fnc8dUqyubSJ1sjTsQX2Y57wrp87nCdbXK2vtoA4Q34cgKhGKQUkoiNvCOLEPKcHIhw1pzAoAblcaIN8JvQTyfOfffaZzWNY0jlEfSlbN3Uda9FWEkhyR24dFgC7IACReI5BABCJSFrHQAuAPK7SYus8A9hufp716xIjKLFczS69L/Y8p6VB7yfyvrBcrZ/EPaGoIqSY/9hx40EiKR5xMyvuRx+J47aAZ8a8Hf6jhqBCoi0SU82TyOFBgKhyZXQ7IDQwYulqQHzhho8kaGuvAOzVPQbWuFIpC91mJEnborhzX1ogMu+55x6VgI2BB8OHD1ceH/0YINEaggrX4ccff2zxXng7Iaqu1R7sGx5CRwCRhUEe8F5hIAJEDgYC6AnoSPp31DlCojsEXWkF9bWcU3ieAQSpNbpIK20/cTyKG2VK3B+WVCCkGDCUG6P5MGIPo6+s0UdU4eZhHsIDeI315u0QugEYmWc9Kg/Cw9VBpWxQ2srrsBclCKwFFUba6eUgXB2UMYBnCULH2iOEc2weKrweUN4AXh2Umli0aJFpvX7NYGSYNcVdMxAAxXnscA4vXLjg0JIbwDpsh/Ap+oXRofb2oGIEKYRiac+F7gm0VWoBnkM9rG8dgi9OFJa2cnyTJk2UsEIo1N7HgLgOFFWEFAOEDypc4wfQVq4G8pyQIwJXvnWOCmoIoaQBwl96PhWGudv6Mcf7Z86cKa4OwqAIvTz55JM2a3AhhGl+g4G9yCExD3XgZv/ss8+a8kdcHQgqhGzhvXjrrbcstiFvyJ75L3otJHhGdVFU3DWDkhIQ+7aAkD116lSxYW2AcCPElS3vKq7bqwEvGcKW1v9M6KIb4gnXCspA6CE1eGggEm2FHuE1suUFKg0IL6IUBsLNs2fPttknlIrQ66jhe4t8x927d8uCBQss2uH7bisvTy/9gXNuHh6GkLPeR3HgeKCMBOra4TtgS1ih1ENxHjTiHjD8R0gJIDQDT0txBQRRtwnbR4wYoUJ7TZs2VSICtZiqVq2qtpsnYGNBjSnUPMINB+IEbeG9cnZIANW80WdbIGkfdYHee+89dWOABwe1p2644QaVSI36U7jRozbSvHnz1HsgvlBvCscHNz2EhpAAjP/2kSfkiGRgRwDBi+rhOAawUa9TtXz5clVzCDWj9PDQ9QAPCupQffPNN/LFF18oTylylVB/Csn+uOHiHOifjba2rhkIeVxj8Hyhr/jnANcxCm6iv8h3wrmuX7++eg3hBoEFbyy8XzNmzFBelZLQw9XIkYJAgVBB/hcEGc45hIdeC0sH1w5sQFh4xYoVqp8QQKh1hfdAuJrXqboWsG8cl4kTJ8r//vc/NR8h9nXy5EnlGYJnDt85fVAHkuhRk+rhhx9W+X16nSqE6FFXzdoLiO8q2kNIYt+wGeII4VmcI0wzVBogKPVpfJACgP0gtwvfib///luFxCHUisv3Im6As4cfEuJqJRWsWb9+vWnIuHWdKnDixAlt+PDhWlRUlBrujUe8xnprzp07p2pSRUdHawEBAVrz5s1VTSsM77e1fwzBx2KL4obO6/bY6qst9HIEJS3mQ8m3bNmiDR48WNmAUgAYVo8h7i+88IK2f/9+i32j5hG2VaxYUbVD+YCjR4+aPtN82HpxQ9Ot+2n+npKGvJdUUqG4Y6rXL7IG5we1wkJCQpQtXbt21X7//Xdt7Nixqv3OnTtLPMZXq1Ol89dff6nSA/Xq1TPVbcJno8wBSlbgs9u1a6ctXry4WPtQvgDHGccbta9sHVPUAUNZC+wT5zAyMlLr2LGj9sorr5SqBAjaYOg/9lG/fn1V/gAlMmrVqqWO05o1a2y+LzU1VdVoQrkRHAccz1atWqnyH+alFkoqC1Hc+bt06ZI2a9YsVdYD/QkMDFQ1zfr166d9/vnnFnWwwN9//63dddddqiwJapzdeeedal1x1xlqsz388MOq5Ab2ffPNN2s//fTTNdWpAqhTh7p2nTt31oKDg9VxwHHr3bu39v7771vUDyPuhxf+OFvYEUKIOwIvHDwLSFa2ziMihHgezKkihJCrgNCRNQjRYXQbRu1RUBFCAD1VhBByFVAbCvlJyJlDjhJGiCEnDKUlIKwwKTYhhFBUEULIVfjnP/+pBiJgYAGKMyKhHyM/kfTduHFjZ3ePEOIiUFQRQgghhNgB5lQRQgghhNgBiipCCCGEEDtAUUXKHUSc27RpoybZLS0oKolq05hE1ijoFbSvpwgmjgf2geNDCCH24qGHHlKFYfXJv0npoKgi5Q6mekBV4ZdfftnZXSElgEreEGy2FlREtwZTkEAoono3KmmjHfZRHKhyjWlBUFU6Ojpa/Pz81JQ+qHJtz+lfiHH+iXBXMOMAym7A/scff7zYdpjyBrMuBAUFqfkG77777hLnyUQFeMxsgOrxeA8qv6OafkmlQTA5d1RUlJrhADMjoMK9rSlzpk6dqiq9W0/PREqG09SQcgXTV+DHFVNB6POCEdclJCRExo8fX2S9LbGEqT0wDQdKDmCaE8wjVxIYObdkyRI19QomDMZks5iqA9OMYPoVTP8CwUWIu//mlcaTDHEzZcoU5R2C8ML0T5hjsVOnTrJmzRo1TY45mF+xV69eShwNHjxYlff4+uuvZdCgQWp6nmeeecaiPb6PmFAb80JiiiNMzYNpl/CZmKIH0/VA9Ok0bNhQfS8x3RCmnIJoI6XAyRXdiYexfPlyNX3DRx99dE3vK2mKEnfF1tQq18q1TklzLZQ0nYstMP3Mxo0b1XQhANNvlPR+TOuxY8eOIusXLVqkbGratGkZe06Mer27I2+88Yaavuo///mPsn/UqFFF2hw6dEi1adiwoZaSkmJaj+mP8D1q0qSJlp+fb1qPKXduuOEGtc18iiS8F/vAlEHW02Rhih18PqbC0SkoKFBTTmH9woULi/Trm2++Uds+/vhjuxwLT4CiipQrmMMMc5slJSXZ3L5nzx6tT58+aj4uzIt1tfm4rvXH/JNPPtGaNWum5t2rU6eO9vbbb5t+XPDjhx8k/FBhPrPPPvvM5v7Onz+vjRs3Tr0fP16YPw3znaGfxc2Thh+uKlWqqDnJunXrpuaNK+kmg+133323Fh4erj4D/fnnP/+pZWRkuKyosuZqoqokcB5gF451WXnrrbfUPpYuXWqxHucO6zH3mq1j+cgjj5jWHTx4UHvuuee0m266Sc35BpsaNGigPf/881p6enqxcwdmZmaq84U5/HCz1Ofn0+eDO3XqlDZkyBB1fnGtYw46iFKwb98+7d5771XXC7bhO3P27FmLzylu3j9zO4qbRzI5OVkbOXKkVr16dWUP5t6zdUMtLbiRYz4/3PhxfWMePdzwcRPXb+z6cbFerK+PhIQEbfz48er9uO5xfO677z6b361rtQfnBN/xFi1aqN8WzKOI9+O7u2vXLs0RYC5M/Nbg+OjnzJaomjRpktpm6zcH1yO24TdBB3MOYh3mGLVm/vz5atv06dNN69LS0tSxwfWI3zpzcI7Q/tZbby2yr+zsbHWcunTpUib7PRGG/0i5gXsKXNaI4yNfwBrMYA8X98WLF+W+++4zzRyPdS1btrzuz0duAPI54NK+7bbblKt83Lhxaub6nTt3qtfIYejRo4dyuw8bNkyFucxDUOfPn1ez1CN/CHlFcLsfP35chasw6/xPP/2k5oMzz2FAe+QmwFXfunVr2b9/v9x+++2qeKQt3n//fRkzZozKk+jbt6+asX7btm0qPIDjhwX5R+VBdna2zJ8/X86cOaPCc+3atVMhBEdToUIF9ejrW/afKP344njdf//9pvV4DXBtoZCnHtbQ15ufl2+++Ub++9//qnU43wjlbNq0SV5//XUVOlm3bp2pr+YMGDBA/vrrL+ndu7c6j3Xr1jVtS05OVtdIZGSkusYOHToky5cvV3lk3333nQqNYyDHo48+Ktu3b1fXZVJSkqxdu1aul5ycHDWtDr5j//jHP5T9yMF58MEHJTExUYV5rvU7jet68+bN6nsKe729vSU2Nla+//579RkIZ+nhLxwz/XsFcGx09O8UwlMYxNKvXz85d+6csh/fK4TArK+9a7EHn4ttLVq0kOHDh4u/v78Kk+G8IzfJHr8x5uTn56vPxO8YQmwbNmwotq2eZ2Zr8A6OL76DOHb6b9HV2gO018H8lPgu43fHPMQHcH7wm4yZAdBnhO918DuDaxHvN/+ukBJwtqojnsPevXvVf0RDhw61uV3/b/aLL76w+V/c9Xqq4GnQvQG6Bwn/DYeEhCjPyLlz50zbNm3apN7Tt29fi33hP0OsR5/M+fHHH9V6eJTM3fS6h23GjBkW7TFLvW6TuacKxwiejZYtW2qJiYkW75k5c6Zqj/+2r+aVQEgAdpd2QWjCGvwXb8u70K5dO+3IkSMO81Rt3rzZ9DnXA/4jh6cDHhQdHFN4Snv06KE+A//x6/zjH/9Q63Bd6MCjhP/WrYEXwNa1ql/D8JZcuHChyPv0Y/j0009brB89erRaHxoaqjxs5jbAi4Vt27dvv25PFdbDU2pu08mTJ7WIiAh1vmDvtbB79261z379+hXZlpWVZeHNu1r4r1OnTpqPj4+2atUqi/XwFsL71bx58zLbA28aznubNm20vLw8i/3gNbxd5lzLdweLrd+lV155RX2Xt23bpl6X5KlCf+GVtAXej/fh+tS5//771Tp939ZgXzExMabX7777bpHfDnPgFcd2899HHVyr2LZ27Vqb7yWWUFSRckN3WU+YMKHIttjYWLUNrnlr8MOMm831iipzd7jObbfdVqzbHa7yWrVqmV7jhxuufNyorcNw4Pbbb1f7WrdunUX7atWqqdCDORBeCCNZ32Seeuopi31YvwehRtwYrnYDRb6SLUFU3GJL/Lz00kvamjVrVEgG9kKo6cID7RFSsLeows2vcePGmre3t11ybxA6Q3/j4+PV66+++sp0zNE/hPF0atasqc55aYBgsg4Vmouq7777zub7sA03POvrB+cb2xD2sg7PfP7552obQtf2EFV//vmnTQFQ0k33aqIKocyrUZKoQm4dtj366KM234vfDGw3DwNeiz2pqammkK/18bXFtXx3bNmEcGKFChUs/vkqSVShbY0aNWz2BflWeN8999xT5Lfm8OHDNt8THR2tQpw6r776aom5rA8++KDabivH8bXXXlPbcB2Sq8PwHyk3Lly4UMTlr4NQCTAPnelgKHKrVq2ueyg29mENhhaXtA1hDR2EZ1CzBaEghAytwfrVq1eryXYRwjl48KBqj1AjRuiYgxAJwiUoK2AOQktAD3dYg1BTacoNINxyvbWrpk2bZvEaxwjlMABG6H300UcyYcIEsReZmZlqVBLsQ6jTVtmGawXnBOEjhHiGDBmiHjFKCtcZRp/qIb8jR46osBOGm5uD++unn36qwi8IT6empqoQoA7CorbAsPjiQDjI+vrRr0OEpqzDM/q24j7rWkA4FeFoa3C9AoTBrwWM8kSfFy1apI4fQnY4b7hWcI2XFv26T0hIUKODrdGveTxitOi12oPQNUoPrFixQoXgH3jgAdVPhLNthW+vZ/Y2hCQR9qtfv36R75A7EhYWph4RTiVXh6KKlBuBgYHq0VYxOdysAPKHbFG9evXr/nz8sFqj5+wUty0vL8/0Oi0trcS+6Dc/vV1ZbELuDICocFVGjRqlRBVyMOwlqnBNINcNImfSpEkyefJku+zXPK9KF1XIS8G5xbZXXnlFnS9b+VTgqaeeknfffVfVz0L9LZxj5OIAlI9Ansq1Xq9luQ6BrVpC10pERIRNsaP3V79mSwv6hlwvCCGIV30YPyacHjt2rJqI2jxHpzj06x55iViKA3k9ZbVn6dKlqi7awoULVb/04438Kqy39Y9SWZg5c6YqDYIcKv1aKU3pkuKOvf57gjbm7UFJ7zHPWy1Ne+vPMP9nB9jr+BgdiipSbuCH1vwH1Bz9y4zEVFvgP1hno9/wiuuLXpdJb1cWm/T34kcOHpWyAm8Z6s6UFngPbdWjsgVuZLZucGUFP9oQVPDyTZw4Ud3g7EXTpk3VDRaiCedh3759KgFcF1AQA6ivpXtBzUUV2s+dO1d5YpCoa35TwbmGqCoOa2+TvdGFhLno1ylJGMHbAE+btRDRr0VbN9WrER4eLu+8847MmTNHeZIgsvAaXhp4gSCSr4Z+3eN9EGOl5VrswfmbMWOGWjC4BNfEvHnz5O2331bX4AcffGBqa8tbVhLwCuvJ9/COoU/F1eHD52DBNa9/R+G9xDWG6woDGMzRvdloo6M/xzYkkpuDfSBx39xbat7eFliPpPRatWoV2ab/Xuu/36RkKKpIuXHjjTeqHz+ExazRR978+eefRbbhBwIiwdk0btxYhfEwUujSpUtF/nPTb8x6KBHF89AeI/fgiTEPAeJH19ZoIIxuQgVlhEMwUqes4HiVdNO3BiOASiuq9JBoSdXSyyKonn32WTWqzt4gzIMio/CuAYRjAW568J5CBOAGixsPKrubV8FGGAijy6zPNYSYM9G9EBhVak1JITyIMNy8rQtJ6vbcdNNNZe4ThCTCgVjg1cMNGiMAdVGle6wwwswafVQf+nYtoqqs9mA0JhZ4L+FJRj/NRdW1fHf0a0z/PuB7q//jYQ5GAiP8iN8R9Ne8b927d1d2/Pzzz2pGAXOQCqC3MW8PjxjaYwTy1drjWodowvcM17S56MdITfwm4x8KW6Nt9d/r5s2bX9Mx8VhKkXdFiN3AqCiM5DEfIaeDUTyOHP1nK0G2pPpXetKxrdF/U6ZMsVi/cuVKm6P/9IJ7pR39h0RcjBhq1KiRSt63BqOUzJNJHVWnCvV1bCXjY31kZGSRujllSVRH8r6ecGtr8EJJ59JWgnZxzJs3T70HSf4YAWp+fnr27KnWYztqHZlz5swZtf7mm2+2eA9GlyGhHNtwjVztmjHH1nuudh5tJaVjEAS+R7DHfJQh6lnpfSuP0X/ot63vztatW9Vn3XLLLUVGoGEQhS06dOigRugtXry4yDYc/99++63M9mBkr61aV6dPn1ZJ4qg552hKSlTHCMdrLf6JQRUlFf+0Pi/FFf/EIAOsL65WWd26dbWoqKjrtt9ToKeKlCtIREZYAJ4YTL9gDkIt+A8O/6nBLa7XqYJnCImnzvYOAL0+EUII8DThP2xMaox8DXgzkNRsHorAFA9IOEedGnjh8N8p6lThP1bUmMF/muYgCfe9996T0aNHq9oxSK694YYb1JQV8JzgsxFqQNjCkaBO17///W+VfwQvFurToJ4S+o3cHngfrKeQQSgG3iYdtMM684T5N954w/RfPKbiwH/OCHcg1Gkr5GIeVgF6kvi11K/SQ3qoMYbrz/z8YNsvv/xi0U4H+VOoN4VcobZt26r6ZQgroaYUnqOukrOA1wE1mBAqReI1vH24Rn744QfloSiub7AJYVuENFEDTa/rhEEkCN9hzsZr9YiiphxCTQi14lzCe4bvL47z008/bWqL4wsPCfLl9u7dq0JzCDvrnikku6MNPC+oKQe74EmMi4tTXhycP+t8zNLagz7huwePONpiPdqgLhiuU/Pr1hnAq43rH78T6COuO32aGoBBIebXLa7/jz/+WNWkwvfQfJoaeJ7wPbP2JOO3CB7ZJ554Ql3zSKTH7wl+i3HsrD1eANcRQqX4PSKlxNmqjngW+M8Q/5GhLo8t8N8k6vJg2Dn+E7d3RfXr9VQBVPlG6QP8p4z/cvFfMerGFFdRHR6nQYMGqbIQqE7ctWvXq1ZU37Jli6rCjqHR+me0bt1ae+GFF5S3yNGeKngFBg4cqMo+YGg2zhk8VKj0bV7byRy9LyUt5se5uCrbJQ1V79+/vyq3gP/srwUMV8f+3nnnHYv1GzZsMH2WXnbBupzHM888ozwZejV1DNfPyclxqqcKwHOBsheoRwTPBDwUmCHg2LFjxXqqsGA2A/MK5KiJVtaK6vAK4ZqENw+lQ9APlCFBFXRMWWSr2jfqTeFzbZXyQN/gBcasB4GBgep3AMccQ/4xZUpZ7YGHF8cKXi14XdBPfLd69+6tvMzlQUmeKh146du2batsR/08/Baa1yezVdMNNuA7ive0b9/epqfP3PuKshU4VjgG+vVsqxYbwDFDnx1Vcd6IeOFPaQUYIfYAlY8xwgf/UV1PMjbxPJD/gvwVeCPItaF7LuBZNQJGs8fVQL4aogXIPbNHNX9PofSFRAixEwidIUEZI30IKS0ImyIEVJrRZISQ6+Ozzz4zhRJJ6WFOFSl3kKODL6wrlEkg7gNGldGxTkj5gPw35HIht42UHob/iNsBdz8qXNuz9hIhRqe04bLS1jjD/q63av/1wPAfcUUoqojbgXpQ1iO1ivOI8QeXkGsD/7CgyvjVwCjD6506ihCjQVFFCCGEEGIHmKhOCCGEEGIHmKhejqBwIWaaRxkBR88NRgghhBD7gKAeCrJiKitbk3jrUFSVIxBUmO2eEEIIIe7HyZMnpWbNmsVup6gqR/RClzgp+qzs9vKAoX4PZhEvSUG7K0a3zxNsNLp9nmAj7XN/jG5jgQPtS0tLU06RqxWspqgqR/SQHwSVvUUV5sTCPo36RTGyfZ5go9Ht8wQbaZ/7Y3QbC8rBvqul7hjvqBJCCCGEOAGKKkIIIYQQO0BRRQghhBBiByiqCCGEEELsAEUVIYQQQogdoKgihBBCCLEDFFWEEEIIIXaAoooQQgghxA5QVBFCCCGEGFVUzZ07V+rUqSMBAQHSoUMH2bJlS4ntly5dKo0bN1btmzdvLitWrCgyEeLUqVMlKipKAgMDpWfPnnL48GGLNklJSTJ06FBViTU0NFQee+wxuXjxomn7Sy+9pCqpWi9BQUF2tp4QQggh7ojLiaolS5bIhAkTZNq0abJjxw5p2bKl9OrVS86dO2ez/YYNG2TIkCFKBO3cuVP69eunlj179pjazJo1S+bMmSPz5s2TzZs3KyGEfaKcvQ4E1d69e2X16tWyfPlyWbdunYwcOdK0/dlnn5X4+HiLpWnTpvLAAw84+IgQQgghxC3QXIz27dtrY8aMMb3Oz8/XoqOjtZkzZ9psP3DgQK1Pnz4W6zp06KCNGjVKPS8oKNAiIyO12bNnm7anpKRo/v7+2qJFi9Trffv2aTgUW7duNbVZuXKl5uXlpZ0+fdrm5+7atUu9Z926daW2LTU1Vb0Hj/YExyg+Pl49GhGj2+cJNhrdPk+wkfa5P0a3Md+B9pX2/u1SEyrn5OTI9u3bZdKkSaZ1mBQR4bqNGzfafA/Ww7NlDrxQy5YtU8+PHz8uZ8+eVfvQCQkJUWFFvHfw4MHqESG/tm3bmtqgPT4bnq3+/fsX+dyPP/5YGjZsKF27di3WnuzsbLWYz3KtT/qIxV5gXwhx2nOfjmbXyRQ5lZwpdzaLFB9vL8PZd60Y3Uaj2+cJNtI+98foNhY40L7S7tOlRFViYqLk5+dL9erVLdbj9YEDB2y+B4LJVnus17fr60pqU61aNYvtvr6+EhYWZmpjDsKGCxYskBdeeKFEe2bOnCnTp08vsv78+fMWoUd7nOzU1FR1MbnDzOMJ6Tky8NM9klegybO3xsj9LS2PvbvbVxaMbqPR7fMEG2mf+2N0GwscaF96err7iSp34dtvv1UHeNiwYSW2g8fN3IsGT1VMTIxUrVpVJcTb80JC0jz26w5flLWxJ5WgAm/8elKeuL2ZoewrC0a30ej2eYKNtM/9MbqNBQ60DwPh3E5URUREiI+PjyQkJFisx+vIyEib78H6ktrrj1iH0X/mbVq1amVqY50In5eXp0YE2vpchP7uvvvuIt4va/z9/dViDU62vU84LiRH7NcRpGXlWby+mJMvwQEVDGNfWTG6jUa3zxNspH3uj9Ft9HKQfaXdn0sdVT8/P2nTpo2sWbPGQnnidceOHW2+B+vN2wOM4NPb161bVwkj8zbwGCFXSm+Dx5SUFJXPpbN27Vr12ci9Mgc5Wr/++qsabUjKRuyFSxavNxy54LS+EEIIIfbCpTxVAOEyhNWQNN6+fXt56623JCMjQ4YPH662P/zww1KjRg2VrwTGjRsn3bt3lzfffFP69Okjixcvlm3btsmHH35oUq3jx4+XGTNmSIMGDZTIevHFFyU6OlqVXgBNmjSR3r17y4gRI1TZhdzcXBk7dqxKYkc7cz755BPl8brzzjvL/dgYhZNJlqLq90PnpXcz255IQgghxF1wOVE1aNAglciNYp1IEkeIbtWqVaZQW1xcnIUbrlOnTrJw4UKZMmWKTJ48WQknjPxr1uxKns7EiROVMEPdKXikunTpovZpHiNF4jmEVI8ePdT+BwwYoGpbmQPP1fz58+WRRx5RYUpSNuKsRNW6Q+dVYiEEMCGEEOKueKGugrM74Skg7IhyDhidYO9EdeSEYQSjq8fJc/MLpPGLqyT/cqK6zi8Tukn9apXd3r6yYnQbjW6fJ9hI+9wfo9tY4ED7Snv/Nt5RJS5NfEpWEUEFfjt43in9IYQQQuwFRRVxWujvtsbVLPKqCCGEEHeGooqUK7FJGRaiKiqkMK9t8/EkyczJd2LPCCGEkOuDooo4zVNVO7yidG9YVT3PySuQTcdYWoEQQoj7QlFFnFZOoVZYRbml0ZUQ4Or9lkVcCSGEEHeCooo4pfAnJlGODg2Ubg0jxM+38DJcsz9BCmwksRNCCCHuAEUVKTdQvSPusqiKDg2QCj7eUtHPV7rUj1DrEtKy5e/TqU7uJSGEEFI2KKpIuZGamSvp2Xmm0J/O7U2vzKH4C0OAhBBC3BSKKuKUOf9qhQWZnvcwK62weh9FFSGEEPeEooo4ZeSfuaeqWnCAtIwJVc8PnE0vMjcgIYQQ4g5QVBGniypwB0OAhBBC3ByKKlJu6Enqeo0qc3o2oagihBDi3lBUEad4qmKsPFUNq1cyea82H0uS1Eu55d4/Qggh5HqgqCLlLqpCAiuoxRwvLy9TCDCvQJOf9511Sh8JIYSQskJRRcoFTEMTn5ppM/Snc1eLKNPzH/+OL7e+EUIIIfaAooqUC6dTMkUvlm4d+tO5KSZUoi9PsLz+SCJDgIQQQtwKiipS/hMpFyOqEAK8s3mhtyo3nyFAQggh7gVFFSkX4i5kFFtOwZy7LosqsIIhQEIIIW4ERRVxeo0q6xBg1OUQ4J8IAWYyBEgIIcQ9oKgi5T9FTTGJ6sDb20vubHYlBMhpawghhLgLFFWkXD1Vvt5eEhUSWGLbPi0iTc9/3H3G4X0jhBBC7AFFFXE4mqaZ5vOrWSVQfLy9Smx/U0wVUwjwj8OJkpSRUy79JIQQQq4HiiricC5k5EhGTr56Xis86KrtEQLs2zLaVAiUNasIIYS4AxRVpJyT1EsO/en0a1XD9Py7XQwBEkIIcX0oqojD0UN/Vxv5Z07T6GBpHFlZPd8RlyInU7Ic1j9CCCHEHlBUkfId+Rd29fCfTr+brnirfjqQZPd+EUIIIfaEooq4TI0qa+5tFS1el3PaV+1PUgnvhBBCiKtCUUXKV1SVUKPKGpRe6FgvXD0/lZotu06mOKR/hBBCiD2gqCIOJ+5y+C88yE8q+fte03v7m4UAv93JhHVCCCGuC0UVcShZuflyNq0wyTzmGkJ/Or2bRYq/b+Flunx3vGTnFZZmIIQQQlwNiiriUE4lZ5Ypn0qnckAFJaxASmau/LyX09YQQghxTSiqiEOJS8owPa99DflU5gxqW9P0fPHWOLv0ixBCCLE3FFWkXPKpyhr+Ax3qhknNUH/1fP2RCxb7JIQQQlwFiiriUOKSri/8B7y8vOSeGyNMr7/cdtIufSOEEELsCUUVcfnwH7irabhpIual209KXn6BXfpHCCGE2AuKKlIuNar8fLyleuWAMu8nIqiC3NaoqnqekJYtvx86b7c+EkIIIfaAooo4DFRA10VVzbBA8b7saSorg9rFmJ4v2sIQICGEENfC5UTV3LlzpU6dOhIQECAdOnSQLVu2lNh+6dKl0rhxY9W+efPmsmLFiiI39qlTp0pUVJQEBgZKz5495fDhwxZtkpKSZOjQoRIcHCyhoaHy2GOPycWLF4vs54033pCGDRuKv7+/1KhRQ1599VU7Wm48zqdnS1ZuYZiudhnzqczp1iBCIoMLvV1rDyTI6ZQr+VqEEEKIs3EpUbVkyRKZMGGCTJs2TXbs2CEtW7aUXr16yblz52y237BhgwwZMkSJoJ07d0q/fv3UsmfPHlObWbNmyZw5c2TevHmyefNmCQoKUvvMyiosSAkgqPbu3SurV6+W5cuXy7p162TkyJEWnzVu3Dj5+OOPlbA6cOCAfP/999K+fXsHHg3PnfOvOHx9vGVw+0JvVYEmsnBz7HXvkxBCCLEbmgvRvn17bcyYMabX+fn5WnR0tDZz5kyb7QcOHKj16dPHYl2HDh20UaNGqecFBQVaZGSkNnv2bNP2lJQUzd/fX1u0aJF6vW/fPszSq23dutXUZuXKlZqXl5d2+vRpUxtfX1/twIED12Vfamqq+iw82hMcp/j4ePXoSny9/aRW+/nlavlo3VG72JeQmqndMOlHtc+bXv5Zy8zJ04yAq55De2F0+zzBRtrn/hjdxnwH2lfa+/e1TcTmQHJycmT79u0yadIk0zpvb28Vrtu4caPN92A9PFvmwAu1bNky9fz48eNy9uxZtQ+dkJAQFVbEewcPHqweEfJr27atqQ3a47Ph2erfv7/88MMPUq9ePeXF6t27twoFog28YGFhYcXalJ2drRadtLQ09VhQUKAWe4F9oU/23Kc9OJF4ZeRfrbDAMvfP3L6ISn6qwjqmrEnKyJHlf52R+1pfmR/QXXHVc2gvjG6fJ9hI+9wfo9tY4ED7SrtPlxFViYmJkp+fL9WrV7dYj9cIt9kCgslWe6zXt+vrSmpTrVo1i+2+vr5KLOltjh07JrGxsSp/6/PPP1f9fPrpp+X++++XtWvXFmvTzJkzZfr06UXWnz9/3iL8aI+TnZqaqi4miEFX4XB8kul5kJZVbBj3Wu27p3GwElXgkz+OSJeaFcTdcdVzaC+Mbp8n2Ej73B+j21jgQPvS09PdS1S5+omCxwmCConq4L///a+0adNGDh48KI0aNbL5PnjdzD1p8FTFxMRI1apVVVK8PfuHApnYryt9Uc5lHDU9b1W/pgT6+djFvh5Vq8qNf8bL3jNpsi/hkpzJ9pNWMaHizrjqObQXRrfPE2ykfe6P0W0scKB9GAznVqIqIiJCfHx8JCHBcsJcvI6MLJxQ1xqsL6m9/oh1GP1n3qZVq1amNtYelLy8PDUiUH8/3gvvlS6oQJMmTdRjXFxcsaIKowSxWIOTbe8TjgvJEfu9HuIuT6ZctbK/BAVUsKt9wzrVkYlf7VbP/7cpTlrXLj4M6y644jm0J0a3zxNspH3uj9Ft9HKQfaXdn8scVT8/P+X5WbNmjYXqxOuOHTvafA/Wm7cHGMGnt69bt64SRuZt4C1CrpTeBo8pKSkqn0sHIT18NnKvQOfOnZXQOnr0iufl0KFD6rF27dp2OgLGIjMnX5VUsNfIP2vuaRktoRULhdry3WfkXJr9wqmEEEJIWXAZUQUQKvvoo4/ks88+k/3798vo0aMlIyNDhg8frrY//PDDFonsKHOwatUqefPNN1Xe1UsvvSTbtm2TsWPHmhTr+PHjZcaMGaoEwt9//632ER0drUov6B4nJJ+PGDFC1cRav369ej+S2NEOICm9devW8uijj6rSDRBgo0aNkttvv93Ce0Vsl1OwR40qawIq+MjgdrXU89x8TeZvOGH3zyCEEELcVlQNGjRI1YFCsU6E53bt2qVEk55ojlBbfHxhgjLo1KmTLFy4UD788ENV0+qrr75SI/+aNWtmajNx4kR58sknVd2pdu3aqaKe2Kd5fHTBggWqgGiPHj3krrvuki5duqh9mrv9MAIQIcpu3bpJnz59lBhbvHhxuR0bdxZVMQ4QVWB45zpSwaewSvsXm2LlYnaeQz6HEEIIKQ1eqKtQqpbkukHoESUdMDrB3onqyAvDKEZXiZP/98/j8sryfer5mw+0lAFtajrEvmeX/iVfbT+lnr94d1N5rEtdcUdc8RzaE6Pb5wk20j73x+g2FjjQvtLev413VIlLEHfhSo2q2uGO8VSBkd3qmZ5/8udxyc03Zv0VQgghrg9FFXGLKWqKo2H1ynJLo6rqOeYCXPH3lfAwIYQQUp5QVBGHiqqACt6qpIIjMfdWffD7MVX4jRBCCClvKKqI3Sko0OTk5RpV8FJhFKYj6VgvXJrXCFHP98WnyZ9HEh36eYQQQogtKKqI3UlIz5KcvAKHh/50INpGdb/irXpn7RGHfyYhhBBiDUUVsTtxFxxfTsGaO5tFSb2qQer5luNJsunYhXL5XEIIIUSHoorYnVgHF/60hY+3lzx5W33T6zlrDpfL5xJCCCE6FFXE7pw0H/nnwHIK1vRtES11Ln/ehqMXZOuJpHL7bEIIIYSiirhtOQVrfH28Zcyt9FYRQghxDhRVxO7EmuVU1axSfqIK9LuphknI/XE4UXbEJZfr5xNCCPFcKKqIw8J/kcEBauLj8qSC8lbdYHr9n9WHyvXzCSGEeC4UVcSuYFLjCxk55R76M6f/TTWlZpVAk7dq41GOBCSEEOJ4KKqIw8oplGeSujl+vt7ydM+GptezfjrAKuuEEEIcDkUVMUSSuq3cqobVK6nnO+NSZPW+BKf1hRBCiGdAUUUcV07BiaIKdaue69XY9Hr2Twclv4DeKkIIIY6DoorYldikDKeH/3R6NqkmrWuFqueHz12Ub3eedmp/CCGEGBuKKmJX4pIKJ1J2tqdKnxNwYu/GFiMBs/PyndonQgghxoWiitiVuAuFnqogPx8JD/Jzdnfk5nrh0r1hVfX8dEqmfLr+hLO7RAghxKBQVBG7gZylU8mZpomU4SlyBV64s7F4X+7Ku2uPyPn0bGd3iRBCiAGhqCJ2Iz41U/IuJ4M7O/RnTpOoYBnUrpapjta/WRCUEEKIA6CoIg6pUVXbyUnq1jxzR0Op5O+rni/ZGif7zqQ5u0uEEEIMBkUVMVyNKltEVPKXJ28rnGwZzrRXlu9jQVBCCCF2haKKOERUIafK1Xikcx2T2Nt47AILghJCCLErFFXEbsSaiara4UHiavj7+sjku5qYXr/y4z7JzGGJBUIIIfaBoorYvZo6Bv3VCC2c0NjV6HVjdelYL1w9P5mUKXN/PeLsLhFCCDEIFFXE7uG/6JBANamxK4IyD6/0u1Eq+BTWWPhg3VE5cu6is7tFCCHEALjmnY+4HamZuZJyKdclk9StqV+tsozoWk89z83X5MVle5i0Tggh5LqhqCKGmki5tDx5WwOpWSXQlLT+3a4zzu4SIYQQN4eiiti/nIKL1aiyRaCfj0y/50bT6xk/7lPeNkIIIaSsUFQRuxB7wb08VaBHk+pyR9Pq6nnixRx5beV+Z3eJEEKIG0NRRQxf+LMkpt1zo5r8GSzaclL+PJzo7C4RQghxUyiqiEfmVOmg9AMmXNZ54ZvdkpGd59Q+EUIIcU8oqohdiE3KUI+VA3wltGIFcSeGdqgtHeqGqeenkjNl9k8Hnd0lQgghbghFFblucvML5ExKlslLhVpQ7oS3t5e8PqCFBFQo/DrM33BCthxPcna3CCGEuBkUVeS6iU/JknzMUuxmoT9z6kQEybN3NDK9fv7r3ZzChhBCyDVBUUXsFvpzl3IKxTG8c11pXStUPT+emMHRgIQQQq4JiirisSP/rPHx9pJZ97c0hQE/2xgrvx485+xuEUIIcRNcUlTNnTtX6tSpIwEBAdKhQwfZsmVLie2XLl0qjRs3Vu2bN28uK1assNiOKUimTp0qUVFREhgYKD179pTDhw9btElKSpKhQ4dKcHCwhIaGymOPPSYXL16ZE+7EiRMqV8h62bRpk3g6RhFVoH61SvLPu5qYXj+3dLdcuJjt1D4RQghxD1xOVC1ZskQmTJgg06ZNkx07dkjLli2lV69ecu6cbY/Bhg0bZMiQIUoE7dy5U/r166eWPXv2mNrMmjVL5syZI/PmzZPNmzdLUFCQ2mdWVmFyNYCg2rt3r6xevVqWL18u69atk5EjRxb5vF9++UXi4+NNS5s2bcTTiTMr/Fk7LEjcnYduri23NqqqnidezJYXvvmbcwMSQghxP1H173//W0aMGCHDhw+Xpk2bKiFUsWJF+eSTT2y2f/vtt6V3797y3HPPSZMmTeSVV16R1q1by7vvvqu242b41ltvyZQpU+Tee++VFi1ayOeffy5nzpyRZcuWqTb79++XVatWyccff6w8Y126dJF33nlHFi9erNqZEx4eLpGRkaalQgX3Kh/gSE8VwmdRoQHi7sADiTBgeJCfer16X4Is3nrS2d0ihBDi4riUqMrJyZHt27er8JyOt7e3er1x40ab78F68/YAXii9/fHjx+Xs2bMWbUJCQpR40tvgESG/tm3bmtqgPT4bni1z7rnnHqlWrZoSXt9//714OhCtuqcKhTQr+LjUJVVmqlb2V2UWdF7+YZ8cOXclHEwIIYRY4ysuRGJiouTn50v16oXzseng9YEDB2y+B4LJVnus17fr60pqA6Fkjq+vr4SFhZnaVKpUSd58803p3LmzEltff/21CjPC2wWhZYvs7Gy16KSlpanHgoICtdgL7Avixp77LC3Jl3Ik/XIF8piwQIf0wVn23da4qgxpH6Omr8nMzZcnFmyXb0d3UpMx2xtnnsPywOj2eYKNtM/9MbqNBQ60r7T7dClR5cpERESoXC+ddu3aqdDg7NmzixVVM2fOlOnTpxdZf/78eYt8Lnuc7NTUVHUxQfCVJ/vOXimnUDXAq9jcN3e1b1S7CNl89Lwcu5AlhxIuyvNfbpcpd9QxlI3lgdHt8wQbaZ/7Y3QbCxxoX3p6uvuJKggXHx8fSUhIsFiP18hfsgXWl9Ref8Q6jP4zb9OqVStTG2sxkJeXp0YEFve5ACFEJLYXx6RJkyyEGDxVMTExUrVqVTXK0J4XEvKAsN/y/qJsjr+Sc9aoZngRj5+72wfm/aOS9Htvg1zKyZfl+y5I96bRMqB1TUPZ6GiMbp8n2Ej73B+j21jgQPtQXcDtRJWfn58aTbdmzRoVWtMPEl6PHTvW5ns6duyoto8fP960DkIH60HdunWVMEIbXURB3CBXavTo0aZ9pKSkqHwufTTf2rVr1WdDOBXHrl27LISaNf7+/mqxBifb3iccF5Ij9ns1Tl2engbUDg9y2Oc7yz7QMDJY/tW/uYxfsku9fvG7vdIypoo0rF7ZMDaWB0a3zxNspH3uj9Ft9HKQfaXdn0uJKgDPzrBhw1TSePv27dXIvYyMDDUaEDz88MNSo0YNFVoD48aNk+7du6t8pz59+qgRe9u2bZMPP/zQdIAhuGbMmCENGjRQIuvFF1+U6Ohok3DDqEGMIMSoQ4w2zM3NVSJu8ODBqh347LPPlOi76aab1OtvvvlGjUjEiEFPxrycgrvXqCqJfjfVkM3HL6j8qqzcAnliwQ75bkxnCfJ3ua8QIYQQJ+Fyd4RBgwapnCMU60SSOLxLKHegJ5rHxcVZKMZOnTrJwoULVcmEyZMnK+GE5PFmzZqZ2kycOFEJM9SdgkcKI/ewT3N33oIFC5SQ6tGjh9r/gAEDVG0rc1CuITY2ViWxo9goamrdf//94skYZYqa0jCt742y62Sq7I9PUyMBn136l7w3tLXbTSBNCCHEMXhprGpYbiDsiHIOSKSzd04VcsKQz1TeLt3Or62V0ymZElqxguyaeodDPsOZ9lmDOQHveedP04jHZ25vKE/2aGAoGx2B0e3zBBtpn/tjdBsLHGhfae/fxjuqpNzIySuQM6mZhg/9mVM3IkjeHtJKdOfUm6sPyS/7LAdKEEII8UwoqkiZOZV8SXQ/p6eIKnBb4+ry7B2NTK+RwH7kXOmG2xJCCDEuFFWkzBhpIuVr5YlbbpA+zQtHfl7MzpMRn2+X1MxcZ3eLEEKIE6GoImXmpAeLKiSnz36ghTSOrGzKtRqzYIfk5huzUjEhhJCrQ1FFykyseTkFg4/8s0VFP1/56OG2UqVi4aTafx5JlCnf7lHVfAkhhHgeFFWkzHhy+E8nJqyiElZ+voVfpSXbTsp7vx11drcIIYQ4AYoqct2iqoKPl0SFBIqn0rZOmLz5QEvT69k/HZTv/7oyfQ8hhBDPgKKKlAmEuHRRVbNKRfHx9uwCmH1bRstzva6MCERh0K0nkpzaJ0IIIeULRRUpExcyctQEw3oIjBSOCBzUNsZUw+v/PtsmhxJYaoEQQjwFiipih3wqzw39WY8InNG/mXRtEKFeo8TCP/672WKUJCGEEONyXaJq06ZNamLjp59+Wg4fPqzWXbp0SXbs2CEXL160Vx+Ji0+kXDssyKl9cSUq+HjL+w+1kRY1Q9TrhLRsJazOp2c7u2uEEEJcUVTl5OTIfffdJ507d5Z//vOfauLhkydPFu7Q21vuuOMOefvtt+3dV+KiniqG/yyp5O8r84e3l3pVC8XmiQuXZNgnWyQti8VBCSHEyJRJVL344ouyfPlyef/99+XgwYMWdXkCAgLkgQcekO+++86e/SQuBssplExYkJ988VgHiQ4JUK/3xafJ/83fJlm5hXlohBBCjEeZRNWiRYtk9OjRMnLkSAkLCyuyvUmTJnLs2DF79I+4QfjPEwt/lobo0ED5/LEOSmCBLSeSZMTnFFaEEGJUyiSqzp07J82bNy92u4+Pj8qtIsb3VIUH+alwF7FN/WqVZP7wdqZj9MfhRHn8i+2SnUdhRQghRqNMoiomJkYOHDhQ7Pb169dL/fr1r6dfxIWBp+VsWpZ6znyqq9OiZqgSVhX9fNTr3w6elye+2EFhRQghBqNMourBBx+UDz74QDZu3GgxnBx89NFH8uWXX8rDDz9sv14Sl+JUstnIP4b+Sl11/dNH2klghUJhtebAORmzYKeqZ0UIIcSDRRVG/HXq1Em6desmt956qxJUKKtQq1YtGTVqlPTu3Vu9JsaESeplo0O9cPnkkXYSUKHwa/fL/gR5ctEOCitCCDEIZRJVfn5+smrVKvn000+lXr160rhxY8nOzpYWLVrI/Pnz5YcfflB5VcSYxJonqVNUXRMdbwiX/w5rJ/6XJ2D+aW+CjPofk9cJIcQIlDnDGN6phx56SC3Es6Cn6vroXD9CPh7WVk1jk51XIL8ePC+Pzt8mr95Zy9ldI4QQUt6eKninvv/++2K3o4YV2hBjYj7tCssplI2uDarKZ4+2l6DLyeubjifJk18fkpRLOc7uGiGEkPIUVSdOnChxGhpsi42NLWufiJuE//x8vaV65cLiluTaubleuCwYcbOEBFZQr/clXJIHP97CKW0IIcTT5v7TR/vZYuvWrRIaGlrWXRMXBtXz9fBfTJVA8fYu/jogV6dVTKgsGXWzRFQqLBB64Gy6DPxgIydhJoQQI4sqzOWHkB4WCKrx48ebXpsv4eHh8tZbb8ldd93l2J4TpwAvCvKAAPOp7EPjyGBZPPJmqV650GN1PDFD+r+3QfacTnV21wghhDgiUb1atWpy4403msJ/NWrUUIs5EFtBQUHSpk0beeKJJ66lH8RNiDXzoNQOL5wwmFw/9SKC5IMHGsszPxyTo+czJPFitgz6YKO891Ab6d6wqrO7RwghxJ6iasiQIWoBqE01ZcoU6dGjR2nfTgw45x+rqduXyGA/WTrqZhn5vx2yLTZZMnLy5bH5W+W1AS3k/jY1nd09Qgghjsip+vXXXymoPBSWU3AsoRX95Iv/6yC9bqyuXucVaPLs0r/k3bWHVT4bIYQQ1+W6ZsLNzc1VcwCmpqZKQUHRqtCouE6MK6o4RY1jCKjgI+8NbSPTf9grn28sHEX7xs+H5Fhihsy8r7n4+7KwLiGEGEZUQUBNmjRJ3nvvPbl0qfhRSvn5rBJtZFEVU4WiylH4eHvJ9HtulKiQQHl9VeHk5d/sOK3KWXzwjzYSUcnf2V0khBBij/Dfv/71L5k9e7aqpv7555+rsMRrr70m8+bNU1PVtGzZUn766aey7Jq4iaiqWtlfAi8XriSOAQM/Rt9yg8x9sLVpvsDtscly77vrZX98mrO7RwghxB6iCvP7DRw4UN5//301eTLAiL8RI0bI5s2b1c1g7dq1Zdk1cWEu5eSZClPWZj5VudGnRZQsHdVJIoMLC62eTsmUAe9vkJ/3nnV21wghhFyvqDp16pTcdttt6rm/f2EYIisryzTZMjxY//vf/8qya+LCnEzKND1nknr50rxmiHw3trO0rBmiXl/KyZdRX2yXOWsOS0EBE9gJIcRtRRUKfOrT1FSqVEmCg4Pl2LFjFm2Sk5Pt00PimvlUFFXlTvXgAFkyqqP0bRmtXmMw4L9XH5L/+3ybpF7KdXb3CCHE4ymTqLrpppvUVDQ6qFuFKurr16+XP/74Q+bMmaPyqoixiL2QYXrOkX/OGxk4Z3Area5XI9Fnilp74Jz0ffdP2XuGFdgJIcTtRNXIkSMlOztbLeDVV1+VlJQUVUKhe/fukpaWJm+++aa9+0qcjPl8dAz/OQ/kLI65tb58Nry9VKlYweRFvO+9DfL19lPO7h4hhHgsZSqpcM8996hFp2nTpnL06FH57bffxMfHRzp16iRhYWH27CdxAVj407Xo1rCq/PBkF3liwQ7ZfSpVzcn4zNK/ZHtcsky9u6nyahFCCHFxT5UtQkJC5N5775W7775bCap169bZa9fExeb9w/B+lFQgzqdmlYry5aiOMqR9LdO6hZvjpN/c9XI4Id2pfSOEEE/DbqJK5/vvv5fOnTurPKuyMnfuXKlTp44EBARIhw4dZMuWLSW2X7p0qTRu3Fi1b968uaxYscJiO+poTZ06VaKioiQwMFB69uwphw8ftmiTlJQkQ4cOVUn3oaGh8thjj5mS8a05cuSIVK5cWbXzFDDC7NTl0X/wUiEERVwDeKRQaX3WgBbi71v4lT5wNl3lWS3eEsfpbQghxBVF1erVq5UnqkmTJirE95///Me0bdmyZdKsWTPp37+/EizTpk0rU4eWLFkiEyZMUO/fsWOHSnjv1auXnDt3zmb7DRs2qImeIYJ27twp/fr1U8uePXtMbWbNmqWS51GcFHW0goKC1D71MhAAgmrv3r3KxuXLlytPG3LHbE3Ng8/r2rWreBIJ6VmSk184FRFDf67JwHYx8v3YLtKweiX1Oiu3QF745m8Zu2inpGVxdCAhhDgcrZT8+OOPmre3t+bl5aVVrVpV8/X1Va9feOEF7amnnlLr69evr73//vtaZmamVlbat2+vjRkzxvQ6Pz9fi46O1mbOnGmz/cCBA7U+ffpYrOvQoYM2atQo9bygoECLjIzUZs+ebdqekpKi+fv7a4sWLVKv9+3bh3/lta1bt5rarFy5Utl0+vRpi31PnDhRe+ihh7RPP/1UCwkJuSbbUlNT1efg0Z7gGMXHx6tHR7HxaKJW+/nlapn+/V6tPCkP+5yNPW28lJ2nTfpmt+l8Yen82hpt24kkzVnwHLo/tM/9MbqN+Q60r7T371J7quDtiY6Oln379imvUWJiotxxxx3KW/Xhhx/Ku+++qyZXfvzxx1UYrizk5OTI9u3bVXhOx9vbW73euHGjzfdgvXl7AC+U3v748eNy9uxZizbI/0JYUW+DR4Ty2rZta2qD9vhseLZ0UCUeoUaEJz07ST3QqX0hJYPpg/7Vv7m8N7S1VA4oHItyKjlTHpi3QWb/dEBy8opOfk4IIaQcR/8htPb888+r3CVdmMyYMUPatWsn06dPlyeeeOK6OwOhhkmYq1evbrEeryHYbAHBZKs91uvb9XUltalWrZrFdl9fX5Vwr7e5cOGCPPLII/LFF1+ovKvSYF52AqDUhD4hNRZ7gX0hb8ae+7QmNvFKjaqYsECHfpYz7HM2jrCx943VpVl0Zxm/5C/ZEZciKLw+99ejqq7Vmw+0lMaRlaW84Dl0f2if+2N0GwscaF9p91lqUZWeni61a9e2WKe/hrAyOpjX8MEHH1S1uErLzJkzleC05vz58xb5XPY42ampqepignfNERyOTzI9D9Kyis1xcwTlYZ+zcZSNfiIyp189+d+2s/LxpjOCtLj98ely79z1MuLmaBnaprr4eDt+0AHPoftD+9wfo9tY4ED7oIHsXqfKesSX/hrz/dmDiIgIVecqISHBYj1eR0ZG2nwP1pfUXn/EOoz+M2/TqlUrUxtrkZCXl6dGBOrvR+gPIxvfeOMN9VpXw/BoIfz56KOPFunbpEmTVNK9uacqJiZGqlatWmpvV2lAP3AusF9HfVHOXTqiHnHKW95QQ/zLsQZSedjnbBxt4/N3V5e7W9eRZ5bulkMJFyU3X5P31p+Wzacuyez7m0ud8CBxJDyH7g/tc3+MbmOBA+0rbVrTNYmqzz//XDZt2mR6DW8LDEA+FUb/mYP1b7/99rXsXomzNm3ayJo1a9QIPv0g4fXYsWNtvqdjx45q+/jx403rMIIP60HdunWVMEIbXURB3CBXavTo0aZ9oCI88rnw+bqIwmcj90rPu0JoUue7776T119/XY0+rFGjhs2+YbJpfcJpc3Cy7X3CcbwdsV+duMvlFCKDAyTQv7CKd3niaPtcAUfb2LxmFVUsFPMFfrjumJo7cHtsstz59p8y4faG8liXuuLr47jjy3Po/tA+98foNno5yL7S7u+aRNXPP/+sFmusBVVZRRWAZ2fYsGEqabx9+/ZqTsGMjAwZPny42v7www8rEYPQGhg3bpyaGgfT4vTp00cWL14s27ZtU94jvR8QXMj/atCggRJZL774okq614UbSkT07t1bhfhQdgFlEyDiBg8erNrpbczBZ+Ago4yE0UnPypWkjBz1nBMpuzf+vj4y6c4m0rNJdXnmy7/UAARUYp+58oD8sPuMvD6ghdwYHeLsbhJCiFtSalFVXoltgwYNUjlHKNaJJHF4l1atWmVKNI+Li7NQjKiXtXDhQpkyZYpMnjxZCSe9ZpbOxIkTlTBD3Sl4pLp06aL2ae7OW7BggRJSPXr0UPsfMGCAqm1FMOdfoZcKsEaVMWhXJ0xWjusqb/58SD7dcFx5rfacTpN73l0vI7vVk3E9GnCaG0IIuUa8UFfhWt9EygbCjhg1iUQ6e+dUIScMIxgd4dJdtSdeHv9ih3r+zO0N5ckeDaQ8cbR9roAzbdwZlywvfP23HDSb1qZOeEX5133NpdMNEXb5DJ5D94f2uT9Gt7HAgfaV9v5tvKNKHFujKpyeKqNxU63CXCvkVfldzqk6ceGSPPjRZhm/eKecS7PfSFVCCDEyFFXkmkQVc6qMiZ+vtzzVo4GsGNdF2tauYlq/bNcZue3N3+W/fx6XvMvTFBFCCLENRRW5KrEXroiq2hRVhqZ+tcry5aiOqiJ7aMXCUZ4Xs/PkleX75O53/pQtx6/UKyOEEGIJRRW5Kicve6qC/HwkLMg+NcmI6+Lt7SUPdqgla5+5RYa0j1G1ycCBs+ky8IONMmHJLoYECSHEBhRVpETyCzQ1b5we+rMuAEuMCwT0zPtayDejO0nzGlfKLHyz87Tc8sZv8s6aw5KVe6V2GyGEeDoUVaREzqRkSh4mjUPoj0nqHpvIvmxMZ5nRr5mEBBaGBC/l5Mubqw/JbW/8Jst2npaCy9cIIYR4MtdU/FPH1pQs5sCbgRpQNWvWlFtuucVU3Zy4b+gPsEaV54L5AR+6ubbc2SxS3vrlsCzcEqe8mGdSs2T8kl3y6YYT8mKfJtK2Tpizu0oIIe4lqjCFS2ZmpirSCapUKRwtlJycrB4x7w7qRVy4cEEJrF69eslXX30lFSvypuzW5RQoqjye8Er+8kq/ZvJwx9ry6or98tvBwt+Av06myP3zNkqfFlEysVcjqe3guQQJIcQw4b+VK1eqOe1eeuklJZz0JTExUaZNmyaBgYGyfv16JbIwJQyql+ORuB+xFjWqeKMkhTSoXlnmD28vnz3aXhpWr2Ra/+PueOnx5u8yZdnfTGYnhHgcZRJVmM7lrrvuUlPJ6F4qEBYWpkQV5tFDG1QfhfDCHHrwVBH3g54qUhLdG1aVFU91VflW4ZdHhiIH74tNcdJt9q/y2soDknop19ndJIQQ1xVVmzZtkpYtWxa7Hds2bNhget21a1dJSEgoWw+JS+RUeXuJ1AgNdHZ3iAvi6+Ot8q1+e+4WNWcgSm+ArNwCmff7Uekya62899tRyeRIQUKIwSmTqAoNDZWff/652O0I98FLpXPx4kW7znVHyr/wZ1RIoKq6TUhxVA6oIE/f3lDWTbxVHu1c1zTlTXpWnrzx8yEZ8Okemb/hBMswEEIMS5nukiNGjJDvvvtO7r//flmzZo3ExsaqBc+xbvny5aqNzooVK6RVq1b27DcpBxC2Sc0sDN0w9EeuJZl9at+m8utzt8jAtjWVlxMkXcqTl5fvl66zflXT3mTmUFwRQoxFmUb/IW8Ko//+85//yLfffmuxzcfHRyZMmKDagKysLHnkkUekRYsW9ukxKTdOJjOfipQdhItn3d9SRna7Qd746YCs2luYAnA+PVtNe/P+b0dlVLd6MvTmWlLRr0w/RYQQ4lKU6ZcMZRJef/11eeaZZ0yeKlC7dm3p0aOHVKtWzdQW9aqGDRtmvx4Tp8z5V4uFP0kZqV+tkrw3tLWs33tCFuxKMomrxIvZqiwD8q5Gdqun8rKC/CmuCCHuy3X9gkE8DRkyxH69IS4FR/4Re9KgakV5b2gdOXTuoryz5ois2BMvmiZyISNHZq48oMTVsE51ZFjHOlKFc0wSQjxNVKWnpysvFepRafh1tKJbt27Xs3viZOKSMkzPOUUNsReNI4Nl7tDWcighXd5Ze0SW7z6jxFXypVxVrf2D34/JoHYx8n9d60rNKrzuCCEGF1Uo9Ik6VF9//bXk5xcmm0JU6ZPt6s/1bcQ9oaeKOJKG1SvLO0NuknE96su7a4/ID7vj1dQ3KL2AUYL/2xQrfVtEyajuN0iTKI4eJoQYVFRhZN8PP/wgTz31lKpBZV4AlBhPVFUO8DVNpEuIvalfrbK8NfgmeeaORmpU4JKtJ5WwgsBatuuMWlBkdFT3etKxXrjpnzdCCDGEqEKNqqefflpmzZpl/x4RlyA3v0DOpGSZQn+8kRFHExNWUV6650Z5qkcD+XzjCflswwkVEgS/HzqvFnishneuI/e0jJaACoVFRgkhxK3rVGFi5Dp16ti/N8RlOJOSqTwFgKE/Up6EBfnJ+J4NZf0Lt8lLfZtaVPLfH58mE7/aLZ1fWytv/nxQEji/ICHE3UXVQw89VKQ+FTFuPhU8CISUN6hd9UjnuvL7c7fI24NbScuYUNM2jBhEkjvE1fjFO+WvkylO7SshhJQ5/Ieq6b///ruaOHnkyJESExOjin5a07p1ax5lA9Soqh0W5NS+EM8Gcwve26qGWnbEJcun60/Iyr/j1cTNeWZ5V61rhcrDHetI72aRDA0SQtxHVHXp0sX0fPXq1UW2c/SfcSZSBgz/EVehda0qaom/q7H8b2OsLNoSZ8q72hGXIjvidkmVHyrIA21jZEj7WlI3gv8QEEJcXFR9+umn9u8JcSlYToG4Mpjge2LvxiqpfdnO08p7dTAhXW2DyPpw3TG1dKkfIUM71JKeTatLhcsTPBNCiEuJKk474znhPx9vL4kODXB2dwixCcJ8g9vXUsVCt55IlgWbY2Xl32clJ79Abf/zSKJaqlb2l8HtYlRb88R3QgixJ5xoi9gM3+rhP9yAkNNCiCuDdIP2dcPUMvXubPlq+ylZuCXO9M8BJnFGYvu7vx5R3iuEB+9oWp25V4SQ8hdVjz76qPrR+vDDD1VCOl5fDbT/73//a48+knIm5VKupGfnqecM/RF3I7ySv6rCPqJrPVl/NFEWbIqT1fsTVIkQTIfzx+FEtQQH+Mo9raJlYNsYaV4jhLXYCCHlI6rWrl0r3t7eUlBQoEQVXl/tB4g/UO5LrHk+Fef8I26Kt7eXdG1QVS2oZ4VK7Uu3n5STSZlqe1pWnnyxKU4tjapXlgfa1pR+N9WQiEr+zu46IcTIourEiRMlvibGgknqxGhUDw5QSe1jb60vm48nKXG14u94ycotzL1CkvuMH/fLaysPyK2Nq8l9N9VQjwwPEkKuBeZUkSKwnAIxsveq4w3hapl+z43y4+54Wbr9lGyPTVbbUfdq9b4EtWDOyzubRUq/VjWkQ71wNWiDEEIcKqouXrwoycnJKrnZmlq1al3v7okTiL2QYXpOUUWMSuWACmo0IJYj5y6q5PZvdpySc+nZant6Vp58ue2UWiKDA6RvyyhVgPTG6GCmNxBC7CeqsrKyZPr06SoR/cKFC8W2Y/FPA4T/mFNFPID61SrJC3c2lmfvaCgbjl6QZbtOy097zkpGTuFv2Nm0LPnoj+NqQdt+raKVwOIUToSQ6xZVTzzxhHz22WfSr18/6dq1q1SpUqUsuyEuip7IG1qxggQHVHB2dwgpN1A+pFvDqmrJ7Jcvv+xPkO92nZbfDp5XoUEAr9YbPx9SS4uaIXJX8yjp0zxKarCeGyEeT5lE1TfffCP/93//Jx988IH9e0ScSnZevpxJLRRVtflfOPFgAv18pG/LaLUkZ+TIj3/HK4GFIqM6u0+lqgUJ7s1qBEu3OpVlYMdKUieiklP7TghxI1GFfAJOlmxMTidnqlo+gKENQgqpEuQnD91cWy2nki/J93+dUUnue8+kmdrsOZ2mlvfWn1YCS/dg1Q7n/IOEeAplElX33nuv/PLLLzJq1Cj794g4FZZTIKRkalapKE/cUl8tJxIzZMWeeFWeAYLKWmDNWnVQJbb3ujFSbm9aXRpHVmaSOyEGpkzzj7z44oty7NgxGTlypGzfvl3Onz8vSUlJRZayMnfuXKlTp44EBARIhw4dZMuWLSW2X7p0qTRu3Fi1b968uaxYscJiO0YmTp06VaKioiQwMFB69uwphw8ftmiD/g4dOlSCg4MlNDRUHnvsMTWyUefgwYNy6623SvXq1dXn1KtXT6ZMmSK5ubliVFFVm0nqhJRInYggJa6WP9lVfn2mmzzRuYbyUpkDb9a/Vx+SO9/+Q7rO+lVe/mGfbDx6QfIuz09ICPFwT1WDBg3U486dO0uciqYso/+WLFkiEyZMkHnz5ilB9dZbb0mvXr2UqKlWrVqR9hs2bJAhQ4bIzJkz5e6775aFCxeqBPodO3ZIs2bNVJtZs2bJnDlzVHJ93bp1lSjEPvft26cEEoCgio+Pl9WrVyuhNHz4cCUasT9QoUIFefjhh1XYE6Lrr7/+khEjRqgq8//617/EKMRdnisNMPxHSOlBmO/hdpHybJ8Wcio5y+TBQs6VzqnkTPlk/XG1YCDIbY2rqTkIkRhf0Y9lAwlxd7w0WwWmrsJLL71UKhf2tGnTrrlDEFLt2rWTd999V72GaImJiZEnn3xSXnjhhSLtBw0aJBkZGbJ8+XLTuptvvllatWqlhBnMi46OlmeeeUaeffZZtT01NVV5nObPny+DBw+W/fv3S9OmTWXr1q3Stm1b1WbVqlVy1113yalTp9T7bQHxh/f88ccfpbItLS1NQkJC1OfDI2YvcIzOnTunRCemE7oeRny+TRU+BH8+f6sKdTgbe9rnqhjdRqPbV5KNp1My5ZfLBUU3HbtgGkVojp+vt3StHyE9m1aXWxtVk8gQ1xtJaPRzaHT7PMHGAgfaV9r79zX/awQvzn333SdhYWFSs2ZNsSc5OTkqnDhp0iTTOhwYhOs2btxo8z1YD3FjDrxQy5YtU8+PHz8uZ8+eVfvQwYGBeMN7IarwCO+TLqgA2uOzN2/eLP379y/yuUeOHFHCC8fCiNXUK/h4SVRIoLO7Q4jbUyM0UIZ1qqOW1Mxc+e3gOfl5b4J61Otg5eQVyJoD59QCmkQFy22NqyqB1SomVJV6IIS4PtcsqiA02rRpI2+++aY89dRTdu1MYmKiChnCi2QOXh84cMDmeyCYbLXHen27vq6kNtahRV9fXyUc9TY6nTp1UqHF7OxsFR58+eWXi7UHbbCYK11dTWOxF9gXPHLXu0/sQ8+pwo3AS7DPa3Zk2h172efKGN1Go9tXWhsr+/tI3xZRakH5kk3HkuTnfQmyZv85UyV3sD8+TS1zfz0qIYEVpGuDCLm1UVXp1iBCwp004bPRz6HR7fMEGwscaF9p93nNosrHx0dq165tIRY8CeR8paenq5yq5557Tt544w2ZOHGizbbI80LleWuQ2I+q9PY82XBJ4mK6HpfnhYxcuXT5P+fISr7KjeoK2Ms+V8boNhrdvrLa2CRUpEmnavJkx6qy7+wl2XgiVTacSJX9CVdyG+HdWr47Xi1IumgaGSSd6gRLp7oh0qhaRfEup9GERj+HRrfPE2wscKB9uO+XhjJlRiK/CTlPGCEHb469iIiIUKItIaEwp0cHryMjI22+B+tLaq8/Yh1G/5m3Qd6V3sZaQOTl5akRgdafi/wugBwseNXgrUK+FvptDcKY5qFJeKrw/qpVq9o9pwo5btjv9VxIp+KuFDWsHxlqc2CAM7CXfa6M0W00un32sDGyushtLQufn0/Plt8PnVeV3P84kqjmIQTwG+89m6GWjzbFS1jFCtLphgjp3CBcutSPUB5mR2H0c2h0+zzBxgIH2qcPanOIqIKY8Pf3lxtuuEHuv/9+Vf4ApQrMgWFPP/30Ne3Xz89PhRbXrFmjRvDpBwmvx44da/M9HTt2VNvHjx9vWocRfFgPMNoPwghtdBEFcYNcqdGjR5v2kZKSovK58Plg7dq16rORe1Uc2I4cMzzaElU4Rliswcm29wnH8b7e/WLEkvlIJlf60tnDPlfH6DYa3T572lg9JFAGtqulltz8AtkRmyy/HoTIOicHzl75jznpUq4s/zteLaBeRJB0aRChBFbHG8LVpNH2xOjn0Oj2eYKNXg6yr7T7K5Oo0kfRgeJKKpRFVAF4doYNG6aSxtu3b69KKmB0H0ocAJQ1qFGjhgqtgXHjxkn37t1VjlefPn1k8eLFsm3bNvnwww9N/YDgmjFjhioFoZdUwIg+Xbg1adJEevfurUokYMQghBJEHJLY9ZF/CxYsUGUVUAcLQgmfAU8URh9ivRGINSunwImUCXENKvh4S4d64WrBpM9nUjKVBwsCC/Wu0rMLvVjgWGKGWj7fGCs+3l4qyR0Cq1vDCGlZkwnvhDiaMokqjKhzFBApyDlCsU4kicO7hFF2eqJ5XFychWJE4jhqSaEQ5+TJk5Vwwsg/vUYVQM4ThBlCdfBIdenSRe3T3J0H0QQh1aNHD7X/AQMGqNpW5onrr7/+uhw6dEjFa5FXhvZlEY6uCqupE+L6RIcGyoMdaqkFBUT/OpUqfx5OlD+PnJedcSmmkg35BZpsj01Wy9trDktlf19pXzdMbq4XrrxYGGEI4UUIcXKdKlI2XL1O1cB5G2XLicJK+Hum95JK/q5RjNDotVU8wUaj2+cqNqZn5crmY0nyx+HCXKxj5zOKbRscAJEVLjfXCysUWZHB4l2CyHIF+xyJ0e3zBBsL3LFOFTEusUmFP8ARlfxcRlARQkoPcqhQQBQLQKgQXiwIrA1HEuVCRo6pbVpWnvyyP0EtAKUbOtQtFFjwZjWqXrlEkUUIKUqZ75y7d++Wd955R9VsgnKzruGAXKajR4+WdfeknMnKzZeEtMIyGZyehhDjhAoHtotRC4ISh89dVFXdkYuFx+RLuRalG1AzCwuoUhEiK1za1Q2TdnWqSOPqlZxoCSEGFlW//fabSuyuUqWKSijHHIC33Xabqr2E6uQ33nijaRQdcQ9OJTOfihAjg390G1avrJaHO9ZRhX0PnUuXTUcvyMZjF2Tz8SRJMRNZEFyr9p5VC6jo5yNNq1eUTg1SVdjwplqhEkSPNiEWlOkbgSTyevXqyaZNm9TUMohfIkkcwgqlCu68806V1E3cc+RfbYoqQgwPQnuNI4PV8kjnukpkoVwDBBa8WJuPXVAhQh0UBt52Ml0tIkdUkvuN0cHStnahJ6tNnSpSrbLrzVlIiMuLKoT8UCkcyVrJycmm2lUAdZ1GjRqlyhZAXBH3wHzkH8N/hHimyGoaHayWx7rUVaMHDyWky7YTSbL1RLJsPZEk8alXatlh++5TqWr5ZH3hiPA64RWlbZ0waVO7ivJkNahWmSMMiUdRJlGF8gKVK1dWzzERMeo0mVckhxdr37599uslcTgsp0AIMQdiCGUXsPxDhQsLZPeRU3L8ordsi02WbSeS5WCC5dQdJy5cUstX20+p10F+PtIyJlQJrJtiqkirWqES4aS5CwlxWVFVv359OXz4sClO37hxY/n2229l6NChat2PP/5Y7LQyxDWJMw//hQc5tS+EENckMthPWtSvJv1b11SvUy/lyva4Qk8WPFp/nUyVnPwrg5YycvJlw9ELajH/p61QZEFsVVGizc/XeMP7iWdSJlF11113ySeffKKqmsNrhSroqHiOwpsAo/70iufEvTxV+HGrVpn/SRJCrk5IxQpyW+PqatFHEe85nSq7TqaoQqQ745LljFnIUP+twfLdrjOm35xm0cFKYKECfIuaIUp44R92QjxCVCFfCtPD6PPdYVoZPP/666/V4z//+U955JFH7N1X4iAw1FoXVTFVAlmbhhBSJgIq+KicKiw6Z1OzZNfJ5MsiK0V2n06RrNwr3qycvALZEZeiFh3UzGpeI0QJLCzNa4ZKdEgAhRYxpqhCDlV4eLjFuoceekgtxP04l54t2XmFP3IM/RFC7ElkSID0DomS3s2i1GtMEH3wbLrsiNOFVrLKwzIHNbP+PIKpdxJN68KD/KQ5RFaNQpEFsVU9mKMNiWtxXUVGsrOz1UhAJKl37txZIiIi7NczUm4wSZ0QUp4TRDerEaKWhzsWrkvKyFHeLIwk/PtUqprPMPFiYTFiHVSDL5xI+rxpXfVgf2leo1BgwbOFkYtIX6BHi7idqMJkwy+99JKqpg5Wr16t6lQlJiaqxPVZs2bJo48+as++knJIUmc5BUJIeRMW5GeRm4WUBMzwsPtUivx9urBsA56bV4AHaJOQdmWqHX2aLSS/3xhdKLKaRgVL3YgglnYgriuqPv30Uxk/frwMHjxY7rjjDgvxBG8VxNXixYspqtyEWDNPFQt/EkKcDTxNCBtGhkTKHTdGmoTW6ZRMU22sv0+nqMd0swKlIPFijvyB+Q4PXwkdBlbwkcZRlVWx0qZRhWKrcWRllQNGiNNF1Ztvvin33nuvLFy4UC5cuDJUVgdT1MCTRdyDk+bhv3CKKkKIawqtmlUqquWu5lEmoYXZIHafTpW9Z1Jl35k02XsmTYUTzcnMzTclyuvAcXVD1UqFQis6WE0gHeGbK1U1rdxtIx4uqo4cOSJPPfVUsdvDwsJsii3imsReyDA9j6lCUUUIcR+hVSciSC33tIy2CB3ui0+VvafTZF98odAyzx0FBZqoCaaxLLtc3gGEVdwvjSKDpVFkZeXNwiPmS+Q8h6Q0lOkqQRV15E4VB6qps/in+xCXlKkekeAZ6Ed3OCHECKHDAFOOFkjLypX9Z66ILHi1Dp9Ll9x8S89U0qVcNf8hFnMwiMdcaOGxTniQ+PqwcCmxQ/HPDz/8UJ544oki2/bu3SsfffQR86nchEs5eaZRNhz5RwgxKsEBFaRDvXC1mNfIgrCCyNofnyZ/x12Q40nZaqShNXrR0tX7riTFo3Bp/aqVTEILS4PqlVlTy4Mpk6iaMWOGmji5WbNm0rdvX3XxfPbZZ6rKOgqARkVFydSpU+3fW+LYcgrMpyKEeBAQRRgliAVzG6I8ULVq1ZS3CrW0DpxNl4Nn09TzQwkXVW6WORBl8HxhMaein4/Ur1ZJLZhUusHl5xhdzVGIxqZMoio6Olq2b98ukydPliVLlqgY9v/+9z81yfKQIUPktddeY80qNyynQE8VIYSgLIO/RNT3l871r9zHCgoKZ54oFFrpcjAhTT0/kZih8rPMuZSTbxqlaC3ikBwPkaWW6oViC0WXUb+LuD9lzryDmv/444/Vcv78eaXyq1atKt7e3pKRkSFnzpxR4ou4Niz8SQghVwfTd+lJ8b2bXckZxnyHR85dVAILocQjCYXJ7yeTL4n1QEJ4thBmxGKOr7eXqqVVKLIqK6FVLyJIrWOCvHthl7MFMWXOW2+9pcJ/+fmWrlLi2qKqNsN/hBByTaDWlV4h3pzMnHw5lnhRCa7DSmilq+eYkiffyrWVV6CZRiKKnLXYFhkcIPWqBqmlbkQl9XhDRCWpUSWQoUQXhBLYwzEXVaymTggh9gEjqfV8LWtv1YkLGUpoKcF1WWwdO58hOflXJprWOZuWpZYNRy1HI/r5eKt/hAsFV6FnSz2PqCRVgvwcbh+xDUWVh6OLKlQcrlrJ39ndIYQQQ4O8KtS9wmJOXn6BnEzOlEMJ6UpgHTt/UY4lFj5aT88DIMCueLeujEgEVSpWUKFDJbaqBqnSDxBgMVUCHW6fp0NR5cHABX3qco0q5FNxCDAhhDgH1LuCEMJiTXJGjklg4fE4RFfiRTmReMmmdwsiLDkuRXaYVZDXCavoK3WrFtbYqhNeUWpHXH4MC5KQihUcZp+nQFHlwSSkZZm+kAz9EUKIa4JwXhsstasU+cf4TEqmHIXYuiy0jivxlSHxqVk295V0KU+SYpNle2xy0c+pWEFq6WLL6hGTXvMfbzuKqh07dpS2qRr5R1wfjvwjhBD3BYnq+IcYyy2NihZ2hriCyMJvPUo/IJfr+PmLkphRNJxo8nBdSpG/Thb1cFX295XaEYUiC/cLTGmmHsMCJTo0kCUhrlVUtW3bttQqFXWrqGjdq0YVR/4RQohxqOjnW2RUol7gNCgkTE6lZKl5XzEaUT0mFj6eKcbDlZ6dJ3tOp6nFGgxCjAoJVAILYqtQ6AWaxFfVyv4eowlKLao+/fRTx/aElDv0VBFCiOeB2ldNooLVYg3qbp1MgsC6pDxb5o+nki8VKXQKsO50SqZaNklSke3+vt5Ss8plkXVZaCkBdvk1phDyOFE1bNgwx/aElDssp0AIIcS67hbmL8RiDcpBQFhhlCKEl1rwOilTPabYGKUIsvMK5Oj5DLXYIiSwghJZNUMrqvpbNUIDTY8QYMGBvm7j6WKiugcTe1lU4VrFfxGEEEJISeUgCss0VLK5PS0r97LYyiwUX5cnodZFGMSVLVIzcyX1dK7N0CKo5O9rIbTMH2uGBqpphVDx3hWgqPJgcJHrFXvx3wkhhBBSVoIDKtgseKrnWp+/mG0SXeZeLgiv+NRMm6FFcDE7Tw4mYL7FdJvbUQg1OjRAJcyHB4gM7OAtXRtWE2dAUeWhpGflSlJGjnrO0B8hhBBH4uXlJdUqB6ilTe2i23PzC+RsalZhblZypuXj5ee2anIBrEfCPRbQ7obq0rWhOAWKKg/FYs4/iipCCCFOpIKPtylx3RYFBZokXsyWU7ZE1+VHeLQAwoLOgqLKw0N/gCP/CCGEuDLe3l5SLThALa1rWRZB1cOLKZdy5O+jp6WFVZHU8oTVujwUi3IKrFFFCCHEzcOLIYEVpGE155ZooKjyUFBzRIeeKkIIIeT6oajyUFj4kxBCCLEvFFUeLqqC/HzURJmEEEIIMaComjt3rtSpU0cCAgKkQ4cOsmXLlhLbL126VBo3bqzaN2/eXFasWFEkgW3q1KkSFRUlgYGB0rNnTzl8+LBFm6SkJBk6dKgEBwdLaGioPPbYY3Lx4kXT9t9++03uvfdetY+goCBp1aqVLFiwQNyRvPwCNVoCYEZyd6lUSwghhLgyLieqlixZIhMmTJBp06bJjh07pGXLltKrVy81CaQtNmzYIEOGDFEiaOfOndKvXz+17Nmzx9Rm1qxZMmfOHJk3b55s3rxZiSLsMyvrysSREFR79+6V1atXy/Lly2XdunUycuRIi89p0aKFfP3117J7924ZPny4PPzww6qtuxGfmiV5l6us1QpjJXVCCCHELmguRvv27bUxY8aYXufn52vR0dHazJkzbbYfOHCg1qdPH4t1HTp00EaNGqWeFxQUaJGRkdrs2bNN21NSUjR/f39t0aJF6vW+ffugMLStW7ea2qxcuVLz8vLSTp8+XWxf77rrLm348OGlti01NVV9Dh7tCY5RfHy8eiwNfx4+r9V+frlaZizfq7k612qfO2J0G41unyfYSPvcH6PbmO9A+0p7/3apOlU5OTmyfft2mTRpkmmdt7e3Ctdt3LjR5nuwHp4tc+CFWrZsmXp+/PhxOXv2rNqHTkhIiAor4r2DBw9Wjwj5tW3b1tQG7fHZ8Gz179/f5menpqZKkyZNirUnOztbLTppaYXzGhUUFKjFXmBfCHGWdp+xF65MahlTJdCufXEE12qfO2J0G41unyfYSPvcH6PbWOBA+0q7T5cSVYmJiZKfny/Vq1e3WI/XBw4csPkeCCZb7bFe366vK6lNtWqW8wT5+vpKWFiYqY01X375pWzdulU++OCDYu2ZOXOmTJ8+vcj68+fPW4Qe7XGyIfBwMUEIXo0DJxNNz4O9c4oNrboK12qfO2J0G41unyfYSPvcH6PbWOBA+9LTbc876NKiyl349ddfVU7VRx99JDfeeGOx7eBxM/eiwVMVExMjVatWVQnx9ryQkGyO/ZbmQkrMOm163uKGaKkWHiSuzLXa544Y3Uaj2+cJNtI+98foNhY40D4MhHM7URURESE+Pj6SkJBgsR6vIyMjbb4H60tqrz9iHUbumbfBCD69jbW3Ji8vT40ItP7c33//Xfr27Sv/+c9/VKJ6Sfj7+6vFGpxse59wXEil3e/JyyP/vL1EalYJcosv17XY564Y3Uaj2+cJNtI+98foNno5yL7S7s+ljqqfn5+0adNG1qxZY6E88bpjx44234P15u0BRvDp7evWrauEkXkbeIyQK6W3wWNKSorK59JZu3at+mzkXpmXVejTp4+8/vrrFiMD3bVGVVRIoPj5utQlQAghhLgtLuWpAgiXDRs2TCWNt2/fXt566y3JyMhQ4TYA71CNGjVUvhIYN26cdO/eXd58800leBYvXizbtm2TDz/80KRax48fLzNmzJAGDRookfXiiy9KdHS0Kr0AkGzeu3dvGTFihCq7kJubK2PHjlVJ7Ginh/zuvvtu9XkDBgww5VpBCCL3yl1IvZQrqZm56nltzvlHCCGEGFdUDRo0SCVyo1gnhAtCdKtWrTIlmsfFxVm44Tp16iQLFy6UKVOmyOTJk5Vwwsi/Zs2amdpMnDhRCTN4l+CR6tKli9qneYwUhTwhpHr06KH2D+GE2lY6n332mVy6dEmJOV3QAQg6eLDcBU5PQwghhDgGL9RVcNC+iRUIO6KcA0Yn2DtRHTlhGMF4tbjvj7vjZczCHer5c70ayZhb64urcy32uStGt9Ho9nmCjbTP/TG6jQUOtK+092/jHVVSIrFJV2pUMfxHCCGE2A+KKg/jJMN/hBBCiEOgqPIwmFNFCCGEOAaKKg8j9kKhqAoO8JXQin7O7g4hhBBiGCiqPIjc/AI5k1JY+LMW86kIIYQQu0JR5UFAUBVcHuvJ0B8hhBBiXyiqPDD0B2qFufZ8f4QQQoi7QVHlQTBJnRBCCHEcFFUeBMspEEIIIY6DospDw38s/EkIIYTYF4oqDwz/+Xh7SVTIlXkPCSGEEHL9UFR5CJjiURdVNUIDxdeHp54QQgixJ7yzegjJl3LlYnaees7QHyGEEGJ/KKo8cORfDJPUCSGEELtDUeUhxF7IMD3nyD9CCCHE/lBUeWA5hdoUVYQQQojdoajyEBj+I4QQQhwLRZUnTlHDRHVCCCHE7lBUeVj4r0rFChIcUMHZ3SGEEEIMB0WVB5Cdly/xaVnqOZPUCSGEEMdAUeUBnErOFE0rfF4rPMjZ3SGEEEIMCUWVhyWp1woLdGpfCCGEEKNCUeVh5RQY/iOEEEIcA0WVp438C2P4jxBCCHEEFFWeFv5jOQVCCCHEIVBUeVD4r4KPl0QGBzi7O4QQQoghoagyOJqmmTxVMVUqio+3l7O7RAghhBgSiiqDk3gxRy7l5KvnnJ6GEEIIcRwUVR5VToGiihBCCHEUFFUGJy4pw/S8NpPUCSGEEIdBUWVw4i5kmp4z/EcIIYQ4Dooqg8PwHyGEEFI+UFR5UPiPoooQQghxHBRVHuKpiqjkJ0H+vs7uDiGEEGJYKKoMTFZuviSkZavnzKcihBBCHAtFlYdMpFyboooQQghxKBRVBoZJ6oQQQogHi6q5c+dKnTp1JCAgQDp06CBbtmwpsf3SpUulcePGqn3z5s1lxYoVRaZpmTp1qkRFRUlgYKD07NlTDh8+bNEmKSlJhg4dKsHBwRIaGiqPPfaYXLx40bQ9KytLHnnkEbV/X19f6devn7gDsReuiCqG/wghhBAPElVLliyRCRMmyLRp02THjh3SsmVL6dWrl5w7d85m+w0bNsiQIUOUCNq5c6cSO1j27NljajNr1iyZM2eOzJs3TzZv3ixBQUFqnxBKOhBUe/fuldWrV8vy5ctl3bp1MnLkSNP2/Px8JcieeuopJcrc0VNVOzzIqX0hhBBCDI/mQrRv314bM2aM6XV+fr4WHR2tzZw502b7gQMHan369LFY16FDB23UqFHqeUFBgRYZGanNnj3btD0lJUXz9/fXFi1apF7v27dPw2HYunWrqc3KlSs1Ly8v7fTp00U+c9iwYdq9995bJvtSU1PVZ+HRnuA4xcfHq0dzHv10i1b7+eVqiU/J1NyV4uwzEka30ej2eYKNtM/9MbqN+Q60r7T3b5cZY5+TkyPbt2+XSZMmmdZ5e3srz9DGjRttvgfr4dkyB16oZcuWqefHjx+Xs2fPWniXQkJCVFgR7x08eLB6RMivbdu2pjZoj8+GZ6t///5ltik7O1stOmlpaeqxoKBALfYC+0KY03qfsRcKa1T5+3pLRFAFu35meVKcfUbC6DYa3T5PsJH2uT9Gt7HAgfaVdp8uI6oSExNVmK169eoW6/H6wIEDNt8DwWSrPdbr2/V1JbWpVq2axXbkTYWFhZnalJWZM2fK9OnTi6w/f/68RfjRHic7NTVVXUwQg2qdpplG/0UF+0li4nlxV2zZZzSMbqPR7fMEG2mf+2N0GwscaF96erp7iSojAq+buScNnqqYmBipWrWqSoq354Xk5eWl9qtfSAlpWZKdD2+lSN2qlYsIR3fCln1Gw+g2Gt0+T7CR9rk/RrexwIH2YTCcW4mqiIgI8fHxkYSEBIv1eB0ZGWnzPVhfUnv9Eesw+s+8TatWrUxtrBPh8/Ly1IjA4j63tPj7+6vFGpxse59wXEjm+z2ZnGWRpO7uXyBr+4yI0W00un2eYCPtc3+MbqOXg+wr7f5c5qj6+flJmzZtZM2aNRaqE687duxo8z1Yb94eYASf3r5u3bpKGJm3gbcIuVJ6GzympKSofC6dtWvXqs9G7pW7whpVhBBCSPniMp4qgFDZsGHDVNJ4+/bt5a233pKMjAwZPny42v7www9LjRo1VK4SGDdunHTv3l3efPNN6dOnjyxevFi2bdsmH374oUmxjh8/XmbMmCENGjRQIuvFF1+U6OhoU62pJk2aSO/evWXEiBGq7EJubq6MHTtWJbGjnc6+fftUMj08WIit7tq1S63XPV6uBkUVIYQQ4sGiatCgQSqJG8U6kSQOwbJq1SpTonlcXJyFC65Tp06ycOFCmTJlikyePFkJJ4z8a9asmanNxIkTlTBD3Sl4pLp06aL2aR4fXbBggRJSPXr0UPsfMGCAqm1lzl133SWxsbGm1zfddJN6REKcKxJ3eeQfqB1OUUUIIYQ4Gi/UVXD4pxBT6BElHTA6wd6J6sgLQzK6Ljrve2+97IhLUc/3v9xbAv18xF2xZZ/RMLqNRrfPE2ykfe6P0W0scKB9pb1/G++oEkVcUqZ6rFbZ360FFSGEEOIuUFQZkIzsPEm8WFh0lKE/QgghpHygqDIgJ5M5kTIhhBBS3lBUGZC4Cxz5RwghhJQ3FFUGxLycAsN/hBBCSPlAUWVAWKOKEEIIKX8oqgwuqphTRQghhJQPFFUGzqkKrOAjVSsVnXuQEEIIIfaHospg5Bdocio50xT6w1Q9hBBCCHE8FFUGIyEtS3LyC9Rzhv4IIYSQ8oOiymDEmpVT4Mg/QgghpPygqDIYJznyjxBCCHEKFFUGIzYpw/ScoooQQggpPyiqDDqRMqjF8B8hhBBSblBUGbRGFQb91QgNdHZ3CCGEEI+BospgxF0oDP9FBgdIQAUfZ3eHEEII8RgoqgxEWlauJF/KVc+ZT0UIIYSULxRVBuKUeT4VRRUhhBBSrlBUGYhYs3IKrFFFCCGElC8UVQatUcVq6oQQQkj5QlFlwJF/gOE/QgghpHyhqDKoqKodHuTUvhBCCCGeBkWVgTh5OVG9kr+vVKlYwdndIYQQQjwKiiqDkFegyemUTFM+lReqfxJCCCGk3KCoMgjn0nOUsAK1mU9FCCGElDsUVQbhdGq26Tnn/COEEELKH4oqA4oqllMghBBCyh+KKgOKKob/CCGEkPKHosognEnNMT1njSpCCCGk/KGoMpinyttLJDo00NndIYQQQjwOiiqDiSoIKj9fnlZCCCGkvOHd1wCkZuZKena+es7QHyGEEOIcKKoMAOf8I4QQQpwPRZUBiLtgJqpYo4oQQghxChRVBiAumZ4qQgghxNlQVBnNU0VRRQghhDgFiioDEJdUOJEyqB0W5NS+EEIIIZ6KS4qquXPnSp06dSQgIEA6dOggW7ZsKbH90qVLpXHjxqp98+bNZcWKFRbbNU2TqVOnSlRUlAQGBkrPnj3l8OHDFm2SkpJk6NChEhwcLKGhofLYY4/JxYsXLdrs3r1bunbtqj4nJiZGZs2aJa7Aycvhv+AAXwmpWMHZ3SGEEEI8EpcTVUuWLJEJEybItGnTZMeOHdKyZUvp1auXnDt3zmb7DRs2yJAhQ5QI2rlzp/Tr108te/bsMbWB+JkzZ47MmzdPNm/eLEFBQWqfWVlZpjYQVHv37pXVq1fL8uXLZd26dTJy5EjT9rS0NLnjjjukdu3asn37dpk9e7a89NJL8uGHH4ozyckrkDMphZ4qhv4IIYQQJ6K5GO3bt9fGjBljep2fn69FR0drM2fOtNl+4MCBWp8+fSzWdejQQRs1apR6XlBQoEVGRmqzZ882bU9JSdH8/f21RYsWqdf79u3TcCi2bt1qarNy5UrNy8tLO336tHr93nvvaVWqVNGys7NNbZ5//nmtUaNGpbYtNTVVfQ4e7cXx8xe12s8vV8vo/23TjAiugfj4ePVoVIxuo9Ht8wQbaZ/7Y3Qb8x1oX2nv377iQuTk5Cgv0KRJk0zrvL29Vbhu48aNNt+D9fBsmQMv1LJly9Tz48ePy9mzZ9U+dEJCQlRYEe8dPHiwekTIr23btqY2aI/Phmerf//+qk23bt3Ez8/P4nNef/11SU5OlipVqhTpW3Z2tlrMvV2goKBALfYg9sKVEGXNKoF2268rAZsQwjWibZ5io9Ht8wQbaZ/7Y3QbCxxoX2n36VKiKjExUfLz86V69eoW6/H6wIEDNt8DwWSrPdbr2/V1JbWpVq2axXZfX18JCwuzaFO3bt0i+9C32RJVM2fOlOnTpxdZf/78eYvQ4/Ww58R50/MqFfKKDZO6M7iYU1NT1ZcFQteIGN1Go9vnCTbSPvfH6DYWONC+9PR09xNVRgMeN3MvGjxVSHCvWrWqSoi3B0M6h0qretVlX+w5ubV5jFSrWlmM+EXx8vJSx82IPwSeYKPR7fMEG2mf+2N0GwscaB8GqLmdqIqIiBAfHx9JSEiwWI/XkZGRNt+D9SW11x+xDqP/zNu0atXK1Mbaw5OXl6dGBJrvx9bnmH+GNf7+/mqxBifbXic8rFKAdLzBT26oXKAElRG/KABfFHseN1fE6DYa3T5PsJH2uT9Gt9HLQfaVdn8udVSRr9SmTRtZs2aNhfLE644dO9p8D9abtwcYwae3R8gOose8DTxGyJXS2+AxJSVF5XPprF27Vn02cq/0NhgRmJuba/E5jRo1shn6I4QQQohn4VKiCiBc9tFHH8lnn30m+/fvl9GjR0tGRoYMHz5cbX/44YctEtnHjRsnq1atkjfffFPlXaHMwbZt22Ts2LEm1Tp+/HiZMWOGfP/99/L333+rfURHR6vSC6BJkybSu3dvGTFihKqJtX79evV+JLGjHXjwwQeV6EPpBpReQOmHt99+u0iSPCGEEEI8E5cK/4FBgwapRG4U60QCOEJ0EE16UnhcXJyFG65Tp06ycOFCmTJlikyePFkaNGigRv41a9bM1GbixIlKmKHuFDxSXbp0Ufs0j5EuWLBACakePXqo/Q8YMEDVtjIfMfjzzz/LmDFjlDcNoUr00byWFSGEEEI8Fy/UVXB2JzwFhB0hzjA6wV6J6gBhSuSEYQSjEePkRrfPE2w0un2eYCPtc3+MbmOBA+0r7f3beEeVEEIIIcQJUFQRQgghhNgBiipCCCGEEDtAUUUIIYQQYgcoqgghhBBC7ABFFSGEEEKIHaCoIoQQQgixAxRVhBBCCCF2gKKKEEIIIcSI09QYGb14PSqz2ruKbHp6upp2x6hVco1snyfYaHT7PMFG2uf+GN3GAgfap9+3rzYJDUVVOYKTDWJiYpzdFUIIIYSU4T6O6WqKg3P/lbOKPnPmjFSuXFm8vLzsqqAh1E6ePGnXOQVdBaPb5wk2Gt0+T7CR9rk/RrcxzYH2QSpBUEVHR5foBaOnqhzBiahZs6bD9o+LyIhfFE+xzxNsNLp9nmAj7XN/jG5jsIPsK8lDpWO8oCohhBBCiBOgqCKEEEIIsQMUVQbA399fpk2bph6NiNHt8wQbjW6fJ9hI+9wfo9vo7wL2MVGdEEIIIcQO0FNFCCGEEGIHKKoIIYQQQuwARRUhhBBCiB2gqCKEEEIIsQMUVQZg7ty5UqdOHTXfUYcOHWTLli3iarz00kuqirz50rhxY9P2rKwsGTNmjISHh0ulSpVkwIABkpCQYLGPuLg46dOnj1SsWFGqVasmzz33nOTl5Vm0+e2336R169Zq9Ef9+vVl/vz5DrFn3bp10rdvX1VdF7YsW7bMYjvGf0ydOlWioqIkMDBQevbsKYcPH7Zok5SUJEOHDlVF6kJDQ+Wxxx6TixcvWrTZvXu3dO3aVZ1bVAqeNWtWkb4sXbpUHUu0ad68uaxYsaJcbHzkkUeKnNPevXu7jY0zZ86Udu3aqRkOcD3169dPDh48aNGmPK9Le3+PS2PfLbfcUuQcPv74425h3/vvvy8tWrQwFXrs2LGjrFy50hDnrrQ2uvP5s8Vrr72mbBg/frz7nkeM/iPuy+LFizU/Pz/tk08+0fbu3auNGDFCCw0N1RISEjRXYtq0adqNN96oxcfHm5bz58+btj/++ONaTEyMtmbNGm3btm3azTffrHXq1Mm0PS8vT2vWrJnWs2dPbefOndqKFSu0iIgIbdKkSaY2x44d0ypWrKhNmDBB27dvn/bOO+9oPj4+2qpVq+xuDz7/n//8p/bNN99g9Kz27bffWmx/7bXXtJCQEG3ZsmXaX3/9pd1zzz1a3bp1tczMTFOb3r17ay1bttQ2bdqk/fHHH1r9+vW1IUOGmLanpqZq1atX14YOHart2bNHW7RokRYYGKh98MEHpjbr169XNs6aNUvZPGXKFK1ChQra33//7XAbhw0bpmwwP6dJSUkWbVzZxl69emmffvqp+txdu3Zpd911l1arVi3t4sWL5X5dOuJ7XBr7unfvrj7L/BzinLiDfd9//732448/aocOHdIOHjyoTZ48WV0XsNfdz11pbXTn82fNli1btDp16mgtWrTQxo0bZ1rvbueRosrNad++vTZmzBjT6/z8fC06OlqbOXOm5mqiCjdXW6SkpKgfiqVLl5rW7d+/X93IN27cqF7ji+Lt7a2dPXvW1Ob999/XgoODtezsbPV64sSJSriZM2jQIHVzcSTWgqOgoECLjIzUZs+ebWGjv7+/Eg0AX2y8b+vWraY2K1eu1Ly8vLTTp0+r1++9955WpUoVk33g+eef1xo1amR6PXDgQK1Pnz4W/enQoYM2atQoh9qoi6p777232Pe4m43nzp1T/f3999/L/bosj++xtX36Tdn8BmaNO9kHcC19/PHHhjt3tmw00vlLT0/XGjRooK1evdrCJnc8jwz/uTE5OTmyfft2FVoyn18Qrzdu3CiuBsJfCCXVq1dPhYTgsgWwITc318IOhHpq1aplsgOPCPtUr17d1KZXr15qAs29e/ea2pjvQ29T3sfi+PHjcvbsWYu+YM4ouJPN7UE4rG3btqY2aI/zt3nzZlObbt26iZ+fn4U9COEkJye7hM1wqcPd3qhRIxk9erRcuHDBtM3dbExNTVWPYWFh5Xpdltf32No+nQULFkhERIQ0a9ZMJk2aJJcuXTJtcxf78vPzZfHixZKRkaFCZEY7d7ZsNNL5GzNmjArfWffDHc8jJ1R2YxITE9UXzfxiAnh94MABcSUgKBDDxs03Pj5epk+frvJo9uzZowQIbqq4AVvbgW0Aj7bs1LeV1AZfrszMTJXbVB7o/bHVF/O+QoyY4+vrq2545m3q1q1bZB/6tipVqhRrs74PR4L8qfvuu0/18ejRozJ58mS588471Y+Qj4+PW9lYUFCg8jg6d+6sbk7655fHdQnx6OjvsS37wIMPPii1a9dW/+wgt+35559Xgvabb75xC/v+/vtvJTCQd4N8m2+//VaaNm0qu3btMsy5K85GI5w/AKG4Y8cO2bp1q1jjjt9BiipSLuBmq4PES4gs/Bh8+eWX5SZ2iH0ZPHiw6Tn+U8R5veGGG5T3qkePHuJO4D9lCPw///xTjEhx9o0cOdLiHGJgBc4dRDLOpauDf9IgoOCF++qrr2TYsGHy+++/i5EozkYIK3c/fydPnpRx48bJ6tWrVXK4EWD4z42ByxceAeuREHgdGRkprgz+82jYsKEcOXJE9RXu15SUlGLtwKMtO/VtJbXBqJnyFG56f0o6L3g8d+6cxXaMVsFoOXvY7Izzj7AurkmcU3eycezYsbJ8+XL59ddfpWbNmqb15XVdOvp7XJx9tsA/O8D8HLqyffBiYCRXmzZt1GjHli1byttvv22Yc1eSjUY4f9u3b1e/ERiVBy82FgjGOXPmqOfwFLnbeaSocmPwZcMXbc2aNRZufrw2j7m7IhhWj/+m8J8VbKhQoYKFHXBhI+dKtwOPcIOb36Tx3w2+FLorHG3M96G3Ke9jgXAWvojmfYGbGXlE5vbghwI/Kjpr165V50//YUQblDVAToG5PfjPFWExV7IZnDp1SuVU4Zy6g43Iv4fgQDgF/bIOQ5bXdemo7/HV7LMFPCLA/By6qn22wH6zs7Pd/tyVxkYjnL8ePXqo/qHf+oIcTOTc6s/d7jxec5o+cSkwDBSjyubPn69GW40cOVINAzUfCeEKPPPMM9pvv/2mHT9+XA2Rx/BXDHvFiCR92CyGe69du1YNm+3YsaNarIfN3nHHHWp4OIbCVq1a1eaw2eeee06NEJk7d67DSipgtAqG72LB1+jf//63eh4bG2sqqYDz8N1332m7d+9Wo+RslVS46aabtM2bN2t//vmnGv1iXm4AI19QbuAf//iHGkKNcw37rMsN+Pr6am+88YayGaMs7VVSoSQbse3ZZ59VI3BwTn/55RetdevWyoasrCy3sHH06NGq7AWuS/Mh6ZcuXTK1Ka/r0hHf46vZd+TIEe3ll19WduEc4lqtV6+e1q1bN7ew74UXXlAjGdF3fMfwGiNLf/75Z7c/d6Wx0d3PX3FYj2h0t/NIUWUAUHMDFx1qbGBYKGoCuRoYvhoVFaX6WKNGDfUaPwo6EBtPPPGEGi6Mi79///7qBmDOiRMntDvvvFPVMYIgg1DLzc21aPPrr79qrVq1Up+DHxjU6XEE+BwIDesFZQb0sgovvviiEgz4ovbo0UPVmTHnwoULSmBUqlRJDf8dPny4EivmoMZVly5d1D5w3CDWrPnyyy+1hg0bKpsxbBh1bRxtI27M+BHDjxcETu3atVVdF+sfIFe20ZZtWMyvmfK8Lu39Pb6afXFxceoGHBYWpo49aojhpmNe58iV7Xv00UfVdYf94TrEd0wXVO5+7kpjo7ufv9KKKnc7j174UzbHHSGEEEII0WFOFSGEEEKIHaCoIoQQQgixAxRVhBBCCCF2gKKKEEIIIcQOUFQRQgghhNgBiipCCCGEEDtAUUUIIYQQYgcoqgghhuSRRx6ROnXqlOm9L730knh5edm9T4QQY0NRRQgpVyBWSrP89ttv4qn88MMP0r17d6lWrZpUrFhRTVY9cOBAWbVqlanNmTNnlPjT53sjhDgfVlQnhJQrX3zxhcXrzz//XE1u+r///c9i/e23365mqS8rmKQZk6L6+/tf83vz8vLUEhAQIOXNG2+8Ic8995wSVffee68SVUeOHJFffvlFWrZsKfPnz1fttm3bJu3atZNPP/1UeeUIIc7H19kdIIR4Fg899JDF602bNilRZb3emkuXLimBUVowu31Z8fX1VUt5AyH3yiuvKEH5888/F9l+7ty5cu8TIaT0MPxHCHE5brnlFmnWrJls375dunXrpsTU5MmT1bbvvvtO+vTpI9HR0coLdcMNNyghkp+fX2JO1YkTJ1RYEZ6gDz/8UL0P74e3Z+vWrVfNqcLrsWPHyrJly1Tf8N4bb7zRIiSng9Bl27ZtlacLn/PBBx+UKk8rMTFR0tLSpHPnzja3Ixyo7x/9BsOHDzeFTHUvFti8ebP07t1bQkJC1PGD52v9+vU27Txw4IAKLwYHB0t4eLiMGzdOsrKySuwrIaQo9FQRQlySCxcuyJ133imDBw9WXiw9FAjhUKlSJZkwYYJ6XLt2rUydOlWJkdmzZ191vwsXLpT09HQZNWqUEhSzZs2S++67T44dO3ZV79aff/4p33zzjTzxxBNSuXJlmTNnjgwYMEDi4uKUGAE7d+5UYiYqKkqmT5+uxN7LL78sVatWvWrfIJoCAwNVTtWTTz4pYWFhNts1adJE7RN2jxw5Urp27arWd+rUST3imODYtWnTRqZNmybe3t4qTHjbbbfJH3/8Ie3bt7fYHwQVBOjMmTOV5xB2JScnq9AsIeQaQE4VIYQ4izFjxiCv02Jd9+7d1bp58+YVaX/p0qUi60aNGqVVrFhRy8rKMq0bNmyYVrt2bdPr48ePq32Gh4drSUlJpvXfffedWv/DDz+Y1k2bNq1In/Daz89PO3LkiGndX3/9pda/8847pnV9+/ZVfTl9+rRp3eHDhzVfX98i+7TF1KlTVbugoCDtzjvv1F599VVt+/btRdpt3bpVtfv0008t1hcUFGgNGjTQevXqpZ6bH7e6detqt99+exE777nnHot9PPHEE2o97COElB6G/wghLgnCawhtWQNPjg48TgiZwVODnCuEsa7GoEGDpEqVKqbXupcHnqqr0bNnTxXO02nRooUKmenvhVcKCeX9+vVT4Umd+vXrK89RaYB3C960m266SX766Sf55z//qTxOrVu3lv3791/1/RgNePjwYXnwwQeVtw/HB0tGRob06NFD1q1bpxL4zRkzZozFa3jJwIoVK0rVZ0JIIQz/EUJckho1aoifn1+R9Xv37pUpU6aoEBdCfuakpqZedb+1atWyeK0LLIS7rvW9+vv19yKRPDMzU4koa2ytK44hQ4aoBfYhNwohTwitvn37yp49e0oclQhBBYYNG1ZsGxwnc2HZoEEDi+0QjggZIg+NEFJ6KKoIIS6JuUdKJyUlRSVcwzuEnCLc/CEwduzYIc8//3wRD4wtfHx8bK4vTXWZ63lvWYCdGAmIBflen332mRJZOAbFoR8D5Je1atXKZhvkopUEC58SUjYoqgghbgNGvSGkhWRxjArUOX78uLgCSDSHyENdKWtsrbsWMJoQoio+Pr5E4aOHJyHIEK4sDfBu1a1b16KvEGdlrUhPiKfCnCpCiNuge4rMPUM5OTny3nvviav0D0IGZRdQ8dxcpKxcufKq70de2MaNG21u09/fqFEj9RgUFGTy3pmD/CsIK5SOuHjxYpH9nD9/vsi6uXPnWrx+55131GNp88AIIYXQU0UIcRtQMgC5QMgXeuqpp5S3BpXYXWliCNR+QuFO1JoaPXq0Sl5/9913VW2rq00pA1EFG2+++WZVliEmJkaJJog0lEJAAjwS2AGEU2hoqMybN0+Vd4DI6tChg/I4ffzxx0oQoY4Wkv2Rn3b69Gn59ddflQcLJRvMgafvnnvuUZ8JUYeq90h0RwV3QkjpoaeKEOI2oBbU8uXLVQ0oJKvDG4N8I9SachXgKYJXCeLvxRdflP/+978q/wsj76427Q1E0kcffSSRkZGqrhTqYWEf8DghR2rJkiWmtnqOFbxjjz/+uEps//33303FUyGOEDKEoMNoPiS7Y79PP/10kc/FfjHa8oUXXpAff/xRFTlFvwkh1wbn/iOEkHIAXiaMXNRH57mKVw0lHBASjIiIcHZ3CHF76KkihBA7g7IK5kBIoeYTPEiEEOPCnCpCCLEz9erVU3MP4jE2Nlbef/99VXNr4sSJzu4aIcSBUFQRQoidQcL3okWL5OzZsypXqWPHjvKvf/2rSJFNQoixYE4VIYQQQogdYE4VIYQQQogdoKgihBBCCLEDFFWEEEIIIXaAoooQQgghxA5QVBFCCCGE2AGKKkIIIYQQO0BRRQghhBBiByiqCCGEEELsAEUVIYQQQohcP/8PDmf8WNod6xsAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 372
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.395046Z",
     "start_time": "2025-03-24T10:58:41.390748Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from torch.optim.lr_scheduler import LambdaLR\n",
    "from torch.optim import Adam\n",
    "\n",
    "\n",
    "def get_optimizer(model, config):\n",
    "    \"\"\"\n",
    "    为Transformer模型创建优化器和学习率调度器\n",
    "    \n",
    "    参数说明：\n",
    "    model : nn.Module\n",
    "        需要优化的PyTorch模型\n",
    "    config : dict\n",
    "        配置字典，需包含以下键：\n",
    "        - beta1: Adam优化器的β₁参数（动量衰减率，默认0.9）\n",
    "        - beta2: Adam优化器的β₂参数（平方梯度衰减率，默认0.98）\n",
    "        - eps: Adam优化器的数值稳定项（默认1e-9）\n",
    "        - d_model: 模型维度（用于Noam调度器）\n",
    "        - warmup_steps: 预热步数（用于Noam调度器）\n",
    "\n",
    "    返回：\n",
    "    optimizer: 配置好的Adam优化器\n",
    "    scheduler: 基于Noam衰减策略的学习率调度器\n",
    "    \"\"\"\n",
    "\n",
    "    # 基础学习率设置为0.1（实际学习率由调度器动态调整）\n",
    "    # 注意：Noam调度器会覆盖此基础值，此处设为任意非零值\n",
    "    base_lr = 0.1\n",
    "    beta1 = config[\"beta1\"]  # Adam 的 beta1\n",
    "    beta2 = config[\"beta2\"]  # Adam 的 beta2\n",
    "    eps = config[\"eps\"]\n",
    "    optimizer = Adam(model.parameters(), lr=base_lr, betas=(beta1, beta2), eps=eps)\n",
    "\n",
    "    # 初始化Noam学习率调度器\n",
    "    lr_scheduler = NoamDecayScheduler(config)\n",
    "\n",
    "    # 将调度器与优化器绑定\n",
    "    # LambdaLR的机制：每个epoch将base_lr乘以调度器返回的因子\n",
    "    scheduler = LambdaLR(optimizer, lr_lambda=lr_scheduler)\n",
    "    return optimizer, scheduler"
   ],
   "id": "ddb5c8d39486c368",
   "outputs": [],
   "execution_count": 373
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## Callback",
   "id": "cd3e6f34d888bfff"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.400562Z",
     "start_time": "2025-03-24T10:58:41.395046Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "\n",
    "class TensorBoardCallback:\n",
    "    def __init__(self, log_dir, flush_secs=10):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            log_dir (str): dir to write log.\n",
    "            flush_secs (int, optional): write to dsk each flush_secs seconds. Defaults to 10.\n",
    "        \"\"\"\n",
    "        self.writer = SummaryWriter(log_dir=log_dir, flush_secs=flush_secs)\n",
    "\n",
    "    def draw_model(self, model, input_shape):\n",
    "        self.writer.add_graph(model, input_to_model=torch.randn(input_shape))\n",
    "\n",
    "    def add_loss_scalars(self, step, loss, val_loss):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/loss\",\n",
    "            tag_scalar_dict={\"loss\": loss, \"val_loss\": val_loss},\n",
    "            global_step=step,\n",
    "        )\n",
    "\n",
    "    def add_acc_scalars(self, step, acc, val_acc):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/accuracy\",\n",
    "            tag_scalar_dict={\"accuracy\": acc, \"val_accuracy\": val_acc},\n",
    "            global_step=step,\n",
    "        )\n",
    "\n",
    "    def add_lr_scalars(self, step, learning_rate):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/learning_rate\",\n",
    "            tag_scalar_dict={\"learning_rate\": learning_rate},\n",
    "            global_step=step,\n",
    "\n",
    "        )\n",
    "\n",
    "    def __call__(self, step, **kwargs):\n",
    "        # add loss\n",
    "        loss = kwargs.pop(\"loss\", None)\n",
    "        val_loss = kwargs.pop(\"val_loss\", None)\n",
    "        if loss is not None and val_loss is not None:\n",
    "            self.add_loss_scalars(step, loss, val_loss)\n",
    "        # add acc\n",
    "        acc = kwargs.pop(\"acc\", None)\n",
    "        val_acc = kwargs.pop(\"val_acc\", None)\n",
    "        if acc is not None and val_acc is not None:\n",
    "            self.add_acc_scalars(step, acc, val_acc)\n",
    "        # add lr\n",
    "        learning_rate = kwargs.pop(\"lr\", None)\n",
    "        if learning_rate is not None:\n",
    "            self.add_lr_scalars(step, learning_rate)\n"
   ],
   "id": "ba2854df2d00ae57",
   "outputs": [],
   "execution_count": 374
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.405036Z",
     "start_time": "2025-03-24T10:58:41.401697Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch.\n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = - np.inf\n",
    "\n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "\n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "\n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ],
   "id": "bf0ca06c7a4f1c7a",
   "outputs": [],
   "execution_count": 375
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.408580Z",
     "start_time": "2025-03-24T10:58:41.405036Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class EarlyStopCallback:\n",
    "    def __init__(self, patience=5, min_delta=0.01):\n",
    "        \"\"\"\n",
    "\n",
    "        Args:\n",
    "            patience (int, optional): Number of epochs with no improvement after which training will be stopped.. Defaults to 5.\n",
    "            min_delta (float, optional): Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute\n",
    "                change of less than min_delta, will count as no improvement. Defaults to 0.01.\n",
    "        \"\"\"\n",
    "        self.patience = patience\n",
    "        self.min_delta = min_delta\n",
    "        self.best_metric = - np.inf\n",
    "        self.counter = 0\n",
    "\n",
    "    def __call__(self, metric):\n",
    "        if metric >= self.best_metric + self.min_delta:\n",
    "            # update best metric\n",
    "            self.best_metric = metric\n",
    "            # reset counter\n",
    "            self.counter = 0\n",
    "        else:\n",
    "            self.counter += 1\n",
    "\n",
    "    @property\n",
    "    def early_stop(self):\n",
    "        return self.counter >= self.patience\n"
   ],
   "id": "2bf81adc3c061003",
   "outputs": [],
   "execution_count": 376
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# 训练模型",
   "id": "d5de37a2206a52d4"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.412315Z",
     "start_time": "2025-03-24T10:58:41.408580Z"
    }
   },
   "cell_type": "code",
   "source": [
    "@torch.no_grad()\n",
    "def evaluating(model, dataloader, loss_fct):\n",
    "    loss_list = []\n",
    "    for batch in dataloader:\n",
    "        encoder_inputs = batch[\"encoder_inputs\"]\n",
    "        encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "        decoder_inputs = batch[\"decoder_inputs\"]\n",
    "        decoder_labels = batch[\"decoder_labels\"]\n",
    "        decoder_labels_mask = batch[\"decoder_labels_mask\"]\n",
    "\n",
    "        # 前向计算\n",
    "        outputs = model(\n",
    "            encoder_inputs=encoder_inputs,\n",
    "            decoder_inputs=decoder_inputs,\n",
    "            encoder_inputs_mask=encoder_inputs_mask\n",
    "        )\n",
    "        logits = outputs.logits\n",
    "        loss = loss_fct(logits, decoder_labels, padding_mask=decoder_labels_mask)  # 验证集损失\n",
    "        loss_list.append(loss.cpu().item())\n",
    "\n",
    "    return np.mean(loss_list)\n"
   ],
   "id": "4ff7dbc94328eee6",
   "outputs": [],
   "execution_count": 377
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.418419Z",
     "start_time": "2025-03-24T10:58:41.412315Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 训练\n",
    "def training(\n",
    "        model,\n",
    "        train_loader,\n",
    "        val_loader,\n",
    "        epoch,\n",
    "        loss_fct,\n",
    "        optimizer,\n",
    "        scheduler=None,\n",
    "        tensorboard_callback=None,\n",
    "        save_ckpt_callback=None,\n",
    "        early_stop_callback=None,\n",
    "        eval_step=500,\n",
    "):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "        \"val\": []\n",
    "    }\n",
    "\n",
    "    global_step = 1\n",
    "    model.train()\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for batch in train_loader:\n",
    "                encoder_inputs = batch[\"encoder_inputs\"]\n",
    "                encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "                decoder_inputs = batch[\"decoder_inputs\"]\n",
    "                decoder_labels = batch[\"decoder_labels\"]\n",
    "                decoder_labels_mask = batch[\"decoder_labels_mask\"]\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # 前向计算\n",
    "                outputs = model(\n",
    "                    encoder_inputs=encoder_inputs,\n",
    "                    decoder_inputs=decoder_inputs,\n",
    "                    encoder_inputs_mask=encoder_inputs_mask\n",
    "                )\n",
    "                logits = outputs.logits\n",
    "                loss = loss_fct(logits, decoder_labels, padding_mask=decoder_labels_mask)\n",
    "\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    "                if scheduler is not None:\n",
    "                    scheduler.step()  # 更新学习率\n",
    "\n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "\n",
    "                # evaluating\n",
    "                if global_step % eval_step == 0:\n",
    "                    model.eval()\n",
    "                    val_loss = evaluating(model, val_loader, loss_fct)\n",
    "                    record_dict[\"val\"].append({\n",
    "                        \"loss\": val_loss, \"step\": global_step\n",
    "                    })\n",
    "                    model.train()\n",
    "\n",
    "                    # 1. 使用 tensorboard 可视化\n",
    "                    cur_lr = optimizer.param_groups[0][\"lr\"] if scheduler is None else scheduler.get_last_lr()[0]\n",
    "                    if tensorboard_callback is not None:\n",
    "                        tensorboard_callback(\n",
    "                            global_step,\n",
    "                            loss=loss, val_loss=val_loss,\n",
    "                            lr=cur_lr,\n",
    "                        )\n",
    "\n",
    "                    # 2. 保存模型权重 save model checkpoint\n",
    "                    if save_ckpt_callback is not None:\n",
    "                        save_ckpt_callback(global_step, model.state_dict(), metric=-val_loss)\n",
    "\n",
    "                    # 3. 早停 Early Stop\n",
    "                    if early_stop_callback is not None:\n",
    "                        early_stop_callback(-val_loss)\n",
    "                        if early_stop_callback.early_stop:\n",
    "                            print(f\"Early stop at epoch {epoch_id} / global_step {global_step}\")\n",
    "                            return record_dict\n",
    "\n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "            pbar.set_postfix({\"epoch\": epoch_id, \"loss\": loss, \"val_loss\": val_loss})\n",
    "\n",
    "    return record_dict\n"
   ],
   "id": "b01763732dc2daa8",
   "outputs": [],
   "execution_count": 378
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.466129Z",
     "start_time": "2025-03-24T10:58:41.419038Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#模型的超参\n",
    "config = {\n",
    "    \"bos_idx\": 1,\n",
    "    \"eos_idx\": 3,\n",
    "    \"pad_idx\": 0,\n",
    "    \"vocab_size\": len(word2idx),\n",
    "    \"max_length\": 128,\n",
    "    \"d_model\": 512,\n",
    "    \"dim_feedforward\": 2048,  # FFN 的隐藏层大小\n",
    "    \"dropout\": 0.1,\n",
    "    \"layer_norm_eps\": 1e-6,  # 层归一化的 epsilon, 防止除零错误\n",
    "    \"num_heads\": 8,\n",
    "    \"num_decoder_layers\": 6,\n",
    "    \"num_encoder_layers\": 6,\n",
    "    \"label_smoothing\": 0.1,\n",
    "    \"beta1\": 0.9,  # Adam 的 beta1\n",
    "    \"beta2\": 0.98,\n",
    "    \"eps\": 1e-9,\n",
    "    \"warmup_steps\": 4_000,\n",
    "    \"share_embedding\": False,  # 是否共享词向量\n",
    "}\n",
    "\n",
    "\n",
    "def get_dl(dataset, batch_size, shuffle=True):\n",
    "    sampler = TransformerBatchSampler(dataset, batch_size=batch_size, shuffle_batch=shuffle)\n",
    "    sample_dl = DataLoader(dataset, batch_sampler=sampler, collate_fn=partial(collate_fct, tokenizer=tokenizer))\n",
    "    return sample_dl\n",
    "\n",
    "\n",
    "# dataset\n",
    "train_ds = LangPairDataset(\"train\", max_length=config[\"max_length\"])\n",
    "val_ds = LangPairDataset(\"val\", max_length=config[\"max_length\"])\n",
    "# tokenizer\n",
    "tokenizer = Tokenizer(word2idx=word2idx, idx2word=idx2word, max_length=config[\"max_length\"])\n",
    "batch_size = 2048\n",
    "# dataloader\n",
    "train_dl = get_dl(train_ds, batch_size=batch_size, shuffle=True)\n",
    "val_dl = get_dl(val_ds, batch_size=batch_size, shuffle=False)"
   ],
   "id": "921d354e93948d1f",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load train dataset from wmt16\\.cache\\de2en_train_128.npy\n",
      "load val dataset from wmt16\\.cache\\de2en_val_128.npy\n"
     ]
    }
   ],
   "execution_count": 379
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:41.962566Z",
     "start_time": "2025-03-24T10:58:41.466129Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#计算模型参数量\n",
    "model = TransformerModel(config)\n",
    "print(f\"模型参数量: {sum(p.numel() for p in model.parameters() if p.requires_grad)}\")"
   ],
   "id": "6885feebc6014036",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "模型参数量: 71938239\n"
     ]
    }
   ],
   "execution_count": 380
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T10:58:42.713835Z",
     "start_time": "2025-03-24T10:58:41.963097Z"
    }
   },
   "cell_type": "code",
   "source": [
    "epoch = 100\n",
    "\n",
    "# model\n",
    "model = TransformerModel(config)\n",
    "# 1. 定义损失函数 采用交叉熵损失\n",
    "loss_fct = CrossEntropyWithPadding(config)\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer, scheduler = get_optimizer(model, config)\n",
    "\n",
    "# 1. tensorboard 可视化\n",
    "if not os.path.exists(\"runs\"):\n",
    "    os.mkdir(\"runs\")\n",
    "exp_name = \"translate-transformer-{}\".format(\"share\" if config[\"share_embedding\"] else \"not-share\")\n",
    "tensorboard_callback = TensorBoardCallback(f\"runs/{exp_name}\")\n",
    "# tensorboard_callback.draw_model(model, [1, MAX_LENGTH])\n",
    "# 2. save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\n",
    "    f\"checkpoints/{exp_name}\", save_step=500, save_best_only=True)\n",
    "# 3. early stop\n",
    "early_stop_callback = EarlyStopCallback(patience=10, min_delta=0.001)\n",
    "\n",
    "model = model.to(device)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# We trained the base models for a total of 100,000 steps or 12 hours. For our big models,(described on the bottom line of table 3), step time was 1.0 seconds. The big models were trained for 300,000 steps (3.5 days)."
   ],
   "id": "a3ee671e3c56f87a",
   "outputs": [],
   "execution_count": 381
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T12:46:04.195628Z",
     "start_time": "2025-03-24T10:58:42.715276Z"
    }
   },
   "cell_type": "code",
   "source": [
    "record = training(\n",
    "    model,\n",
    "    train_dl,\n",
    "    val_dl,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    scheduler,\n",
    "    tensorboard_callback=tensorboard_callback,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    early_stop_callback=early_stop_callback,\n",
    "    eval_step=500\n",
    ")"
   ],
   "id": "fd3f1570d10ee75c",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/1400 [00:00<?, ?it/s]C:\\Program Files\\Python312\\Lib\\site-packages\\torch\\nn\\_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.\n",
      "  warnings.warn(warning.format(ret))\n",
      "38499it [1:47:21,  5.98it/s, epoch=35, loss=2.2, val_loss=3.49]                       "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stop at epoch 36 / global_step 38500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "execution_count": 382
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-24T12:46:04.318433Z",
     "start_time": "2025-03-24T12:46:04.200657Z"
    }
   },
   "cell_type": "code",
   "source": "record",
   "id": "681fbcb793a83bc1",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': [{'loss': 9.85229778289795, 'step': 1},\n",
       "  {'loss': 9.839420318603516, 'step': 2},\n",
       "  {'loss': 9.839776992797852, 'step': 3},\n",
       "  {'loss': 9.83774471282959, 'step': 4},\n",
       "  {'loss': 9.836383819580078, 'step': 5},\n",
       "  {'loss': 9.816690444946289, 'step': 6},\n",
       "  {'loss': 9.816488265991211, 'step': 7},\n",
       "  {'loss': 9.839317321777344, 'step': 8},\n",
       "  {'loss': 9.828093528747559, 'step': 9},\n",
       "  {'loss': 9.857132911682129, 'step': 10},\n",
       "  {'loss': 9.832460403442383, 'step': 11},\n",
       "  {'loss': 9.840683937072754, 'step': 12},\n",
       "  {'loss': 9.814164161682129, 'step': 13},\n",
       "  {'loss': 9.836278915405273, 'step': 14},\n",
       "  {'loss': 9.805928230285645, 'step': 15},\n",
       "  {'loss': 9.82718563079834, 'step': 16},\n",
       "  {'loss': 9.808282852172852, 'step': 17},\n",
       "  {'loss': 9.823722839355469, 'step': 18},\n",
       "  {'loss': 9.80525016784668, 'step': 19},\n",
       "  {'loss': 9.823195457458496, 'step': 20},\n",
       "  {'loss': 9.804931640625, 'step': 21},\n",
       "  {'loss': 9.799339294433594, 'step': 22},\n",
       "  {'loss': 9.775445938110352, 'step': 23},\n",
       "  {'loss': 9.801029205322266, 'step': 24},\n",
       "  {'loss': 9.78722095489502, 'step': 25},\n",
       "  {'loss': 9.789249420166016, 'step': 26},\n",
       "  {'loss': 9.780976295471191, 'step': 27},\n",
       "  {'loss': 9.772008895874023, 'step': 28},\n",
       "  {'loss': 9.766899108886719, 'step': 29},\n",
       "  {'loss': 9.765364646911621, 'step': 30},\n",
       "  {'loss': 9.757120132446289, 'step': 31},\n",
       "  {'loss': 9.780362129211426, 'step': 32},\n",
       "  {'loss': 9.749403953552246, 'step': 33},\n",
       "  {'loss': 9.789969444274902, 'step': 34},\n",
       "  {'loss': 9.784711837768555, 'step': 35},\n",
       "  {'loss': 9.705219268798828, 'step': 36},\n",
       "  {'loss': 9.746829986572266, 'step': 37},\n",
       "  {'loss': 9.757467269897461, 'step': 38},\n",
       "  {'loss': 9.730628967285156, 'step': 39},\n",
       "  {'loss': 9.700901985168457, 'step': 40},\n",
       "  {'loss': 9.718947410583496, 'step': 41},\n",
       "  {'loss': 9.70797348022461, 'step': 42},\n",
       "  {'loss': 9.7308349609375, 'step': 43},\n",
       "  {'loss': 9.710692405700684, 'step': 44},\n",
       "  {'loss': 9.691629409790039, 'step': 45},\n",
       "  {'loss': 9.708917617797852, 'step': 46},\n",
       "  {'loss': 9.67949390411377, 'step': 47},\n",
       "  {'loss': 9.694952964782715, 'step': 48},\n",
       "  {'loss': 9.678803443908691, 'step': 49},\n",
       "  {'loss': 9.685752868652344, 'step': 50},\n",
       "  {'loss': 9.666908264160156, 'step': 51},\n",
       "  {'loss': 9.675345420837402, 'step': 52},\n",
       "  {'loss': 9.651141166687012, 'step': 53},\n",
       "  {'loss': 9.651103973388672, 'step': 54},\n",
       "  {'loss': 9.586955070495605, 'step': 55},\n",
       "  {'loss': 9.626577377319336, 'step': 56},\n",
       "  {'loss': 9.650138854980469, 'step': 57},\n",
       "  {'loss': 9.604105949401855, 'step': 58},\n",
       "  {'loss': 9.61202621459961, 'step': 59},\n",
       "  {'loss': 9.613043785095215, 'step': 60},\n",
       "  {'loss': 9.607662200927734, 'step': 61},\n",
       "  {'loss': 9.575193405151367, 'step': 62},\n",
       "  {'loss': 9.550936698913574, 'step': 63},\n",
       "  {'loss': 9.621129989624023, 'step': 64},\n",
       "  {'loss': 9.515168190002441, 'step': 65},\n",
       "  {'loss': 9.519795417785645, 'step': 66},\n",
       "  {'loss': 9.575667381286621, 'step': 67},\n",
       "  {'loss': 9.590879440307617, 'step': 68},\n",
       "  {'loss': 9.477387428283691, 'step': 69},\n",
       "  {'loss': 9.57374382019043, 'step': 70},\n",
       "  {'loss': 9.534974098205566, 'step': 71},\n",
       "  {'loss': 9.473955154418945, 'step': 72},\n",
       "  {'loss': 9.490764617919922, 'step': 73},\n",
       "  {'loss': 9.541709899902344, 'step': 74},\n",
       "  {'loss': 9.58454704284668, 'step': 75},\n",
       "  {'loss': 9.501103401184082, 'step': 76},\n",
       "  {'loss': 9.503684997558594, 'step': 77},\n",
       "  {'loss': 9.506953239440918, 'step': 78},\n",
       "  {'loss': 9.424301147460938, 'step': 79},\n",
       "  {'loss': 9.474211692810059, 'step': 80},\n",
       "  {'loss': 9.448335647583008, 'step': 81},\n",
       "  {'loss': 9.477253913879395, 'step': 82},\n",
       "  {'loss': 9.433765411376953, 'step': 83},\n",
       "  {'loss': 9.418726921081543, 'step': 84},\n",
       "  {'loss': 9.38894271850586, 'step': 85},\n",
       "  {'loss': 9.384532928466797, 'step': 86},\n",
       "  {'loss': 9.392468452453613, 'step': 87},\n",
       "  {'loss': 9.33521556854248, 'step': 88},\n",
       "  {'loss': 9.5232515335083, 'step': 89},\n",
       "  {'loss': 9.438976287841797, 'step': 90},\n",
       "  {'loss': 9.444611549377441, 'step': 91},\n",
       "  {'loss': 9.434558868408203, 'step': 92},\n",
       "  {'loss': 9.366888046264648, 'step': 93},\n",
       "  {'loss': 9.432214736938477, 'step': 94},\n",
       "  {'loss': 9.41191577911377, 'step': 95},\n",
       "  {'loss': 9.428230285644531, 'step': 96},\n",
       "  {'loss': 9.369425773620605, 'step': 97},\n",
       "  {'loss': 9.33539867401123, 'step': 98},\n",
       "  {'loss': 9.26070499420166, 'step': 99},\n",
       "  {'loss': 9.282228469848633, 'step': 100},\n",
       "  {'loss': 9.301523208618164, 'step': 101},\n",
       "  {'loss': 9.443826675415039, 'step': 102},\n",
       "  {'loss': 9.319476127624512, 'step': 103},\n",
       "  {'loss': 9.342794418334961, 'step': 104},\n",
       "  {'loss': 9.31010627746582, 'step': 105},\n",
       "  {'loss': 9.382492065429688, 'step': 106},\n",
       "  {'loss': 9.35874080657959, 'step': 107},\n",
       "  {'loss': 9.320942878723145, 'step': 108},\n",
       "  {'loss': 9.325387954711914, 'step': 109},\n",
       "  {'loss': 9.211246490478516, 'step': 110},\n",
       "  {'loss': 9.291887283325195, 'step': 111},\n",
       "  {'loss': 9.411750793457031, 'step': 112},\n",
       "  {'loss': 9.266457557678223, 'step': 113},\n",
       "  {'loss': 9.263134956359863, 'step': 114},\n",
       "  {'loss': 9.244681358337402, 'step': 115},\n",
       "  {'loss': 9.231847763061523, 'step': 116},\n",
       "  {'loss': 9.32716178894043, 'step': 117},\n",
       "  {'loss': 9.386195182800293, 'step': 118},\n",
       "  {'loss': 9.231660842895508, 'step': 119},\n",
       "  {'loss': 9.201074600219727, 'step': 120},\n",
       "  {'loss': 9.263223648071289, 'step': 121},\n",
       "  {'loss': 9.268802642822266, 'step': 122},\n",
       "  {'loss': 9.332904815673828, 'step': 123},\n",
       "  {'loss': 9.158123016357422, 'step': 124},\n",
       "  {'loss': 9.19609546661377, 'step': 125},\n",
       "  {'loss': 9.241497993469238, 'step': 126},\n",
       "  {'loss': 9.178193092346191, 'step': 127},\n",
       "  {'loss': 9.227091789245605, 'step': 128},\n",
       "  {'loss': 9.183090209960938, 'step': 129},\n",
       "  {'loss': 9.199774742126465, 'step': 130},\n",
       "  {'loss': 9.107687950134277, 'step': 131},\n",
       "  {'loss': 9.24448299407959, 'step': 132},\n",
       "  {'loss': 9.247517585754395, 'step': 133},\n",
       "  {'loss': 9.286738395690918, 'step': 134},\n",
       "  {'loss': 9.147316932678223, 'step': 135},\n",
       "  {'loss': 9.299941062927246, 'step': 136},\n",
       "  {'loss': 9.139602661132812, 'step': 137},\n",
       "  {'loss': 9.103853225708008, 'step': 138},\n",
       "  {'loss': 9.26478385925293, 'step': 139},\n",
       "  {'loss': 9.198515892028809, 'step': 140},\n",
       "  {'loss': 9.243791580200195, 'step': 141},\n",
       "  {'loss': 9.205801010131836, 'step': 142},\n",
       "  {'loss': 9.141586303710938, 'step': 143},\n",
       "  {'loss': 9.112235069274902, 'step': 144},\n",
       "  {'loss': 9.290319442749023, 'step': 145},\n",
       "  {'loss': 9.249798774719238, 'step': 146},\n",
       "  {'loss': 9.170635223388672, 'step': 147},\n",
       "  {'loss': 9.135482788085938, 'step': 148},\n",
       "  {'loss': 9.120832443237305, 'step': 149},\n",
       "  {'loss': 9.130875587463379, 'step': 150},\n",
       "  {'loss': 9.148263931274414, 'step': 151},\n",
       "  {'loss': 9.150918006896973, 'step': 152},\n",
       "  {'loss': 9.244366645812988, 'step': 153},\n",
       "  {'loss': 9.21929931640625, 'step': 154},\n",
       "  {'loss': 9.181876182556152, 'step': 155},\n",
       "  {'loss': 9.081916809082031, 'step': 156},\n",
       "  {'loss': 9.005908012390137, 'step': 157},\n",
       "  {'loss': 9.119606018066406, 'step': 158},\n",
       "  {'loss': 9.226988792419434, 'step': 159},\n",
       "  {'loss': 9.095703125, 'step': 160},\n",
       "  {'loss': 8.976038932800293, 'step': 161},\n",
       "  {'loss': 9.235550880432129, 'step': 162},\n",
       "  {'loss': 9.151619911193848, 'step': 163},\n",
       "  {'loss': 9.088407516479492, 'step': 164},\n",
       "  {'loss': 9.120441436767578, 'step': 165},\n",
       "  {'loss': 9.077360153198242, 'step': 166},\n",
       "  {'loss': 9.19039249420166, 'step': 167},\n",
       "  {'loss': 8.977577209472656, 'step': 168},\n",
       "  {'loss': 8.96493911743164, 'step': 169},\n",
       "  {'loss': 9.076353073120117, 'step': 170},\n",
       "  {'loss': 8.978434562683105, 'step': 171},\n",
       "  {'loss': 9.133639335632324, 'step': 172},\n",
       "  {'loss': 9.052752494812012, 'step': 173},\n",
       "  {'loss': 8.928984642028809, 'step': 174},\n",
       "  {'loss': 9.117382049560547, 'step': 175},\n",
       "  {'loss': 9.014163970947266, 'step': 176},\n",
       "  {'loss': 9.01236629486084, 'step': 177},\n",
       "  {'loss': 8.950167655944824, 'step': 178},\n",
       "  {'loss': 9.153877258300781, 'step': 179},\n",
       "  {'loss': 9.07026195526123, 'step': 180},\n",
       "  {'loss': 9.110406875610352, 'step': 181},\n",
       "  {'loss': 9.1090669631958, 'step': 182},\n",
       "  {'loss': 9.095852851867676, 'step': 183},\n",
       "  {'loss': 9.136886596679688, 'step': 184},\n",
       "  {'loss': 9.012982368469238, 'step': 185},\n",
       "  {'loss': 8.964226722717285, 'step': 186},\n",
       "  {'loss': 8.920334815979004, 'step': 187},\n",
       "  {'loss': 9.035633087158203, 'step': 188},\n",
       "  {'loss': 9.074846267700195, 'step': 189},\n",
       "  {'loss': 8.943182945251465, 'step': 190},\n",
       "  {'loss': 8.846877098083496, 'step': 191},\n",
       "  {'loss': 8.90821361541748, 'step': 192},\n",
       "  {'loss': 8.754222869873047, 'step': 193},\n",
       "  {'loss': 9.05158805847168, 'step': 194},\n",
       "  {'loss': 9.072781562805176, 'step': 195},\n",
       "  {'loss': 8.9912691116333, 'step': 196},\n",
       "  {'loss': 8.882022857666016, 'step': 197},\n",
       "  {'loss': 9.067137718200684, 'step': 198},\n",
       "  {'loss': 8.911053657531738, 'step': 199},\n",
       "  {'loss': 8.83899974822998, 'step': 200},\n",
       "  {'loss': 9.137727737426758, 'step': 201},\n",
       "  {'loss': 9.206663131713867, 'step': 202},\n",
       "  {'loss': 8.912630081176758, 'step': 203},\n",
       "  {'loss': 9.078276634216309, 'step': 204},\n",
       "  {'loss': 8.956127166748047, 'step': 205},\n",
       "  {'loss': 9.07104778289795, 'step': 206},\n",
       "  {'loss': 9.123332977294922, 'step': 207},\n",
       "  {'loss': 8.825067520141602, 'step': 208},\n",
       "  {'loss': 8.915642738342285, 'step': 209},\n",
       "  {'loss': 9.041122436523438, 'step': 210},\n",
       "  {'loss': 8.927345275878906, 'step': 211},\n",
       "  {'loss': 8.929030418395996, 'step': 212},\n",
       "  {'loss': 8.908472061157227, 'step': 213},\n",
       "  {'loss': 8.771285057067871, 'step': 214},\n",
       "  {'loss': 8.981488227844238, 'step': 215},\n",
       "  {'loss': 9.137718200683594, 'step': 216},\n",
       "  {'loss': 9.013188362121582, 'step': 217},\n",
       "  {'loss': 8.961217880249023, 'step': 218},\n",
       "  {'loss': 8.769502639770508, 'step': 219},\n",
       "  {'loss': 8.916037559509277, 'step': 220},\n",
       "  {'loss': 8.831636428833008, 'step': 221},\n",
       "  {'loss': 8.87822151184082, 'step': 222},\n",
       "  {'loss': 8.862645149230957, 'step': 223},\n",
       "  {'loss': 8.944973945617676, 'step': 224},\n",
       "  {'loss': 9.0821533203125, 'step': 225},\n",
       "  {'loss': 8.946673393249512, 'step': 226},\n",
       "  {'loss': 8.919445991516113, 'step': 227},\n",
       "  {'loss': 8.948383331298828, 'step': 228},\n",
       "  {'loss': 8.845650672912598, 'step': 229},\n",
       "  {'loss': 8.829924583435059, 'step': 230},\n",
       "  {'loss': 8.929231643676758, 'step': 231},\n",
       "  {'loss': 9.007211685180664, 'step': 232},\n",
       "  {'loss': 8.880855560302734, 'step': 233},\n",
       "  {'loss': 8.76544189453125, 'step': 234},\n",
       "  {'loss': 8.983942031860352, 'step': 235},\n",
       "  {'loss': 8.91183853149414, 'step': 236},\n",
       "  {'loss': 8.89694595336914, 'step': 237},\n",
       "  {'loss': 8.818755149841309, 'step': 238},\n",
       "  {'loss': 8.951831817626953, 'step': 239},\n",
       "  {'loss': 8.868741989135742, 'step': 240},\n",
       "  {'loss': 8.705134391784668, 'step': 241},\n",
       "  {'loss': 8.788468360900879, 'step': 242},\n",
       "  {'loss': 8.729637145996094, 'step': 243},\n",
       "  {'loss': 8.829389572143555, 'step': 244},\n",
       "  {'loss': 8.729175567626953, 'step': 245},\n",
       "  {'loss': 9.03392219543457, 'step': 246},\n",
       "  {'loss': 8.755537033081055, 'step': 247},\n",
       "  {'loss': 8.973464965820312, 'step': 248},\n",
       "  {'loss': 8.903894424438477, 'step': 249},\n",
       "  {'loss': 8.819145202636719, 'step': 250},\n",
       "  {'loss': 8.888392448425293, 'step': 251},\n",
       "  {'loss': 8.67601203918457, 'step': 252},\n",
       "  {'loss': 8.678215026855469, 'step': 253},\n",
       "  {'loss': 8.900745391845703, 'step': 254},\n",
       "  {'loss': 8.909942626953125, 'step': 255},\n",
       "  {'loss': 8.74135684967041, 'step': 256},\n",
       "  {'loss': 8.811885833740234, 'step': 257},\n",
       "  {'loss': 8.828791618347168, 'step': 258},\n",
       "  {'loss': 8.825249671936035, 'step': 259},\n",
       "  {'loss': 8.82357120513916, 'step': 260},\n",
       "  {'loss': 8.596879005432129, 'step': 261},\n",
       "  {'loss': 8.817115783691406, 'step': 262},\n",
       "  {'loss': 8.659032821655273, 'step': 263},\n",
       "  {'loss': 8.916333198547363, 'step': 264},\n",
       "  {'loss': 8.824317932128906, 'step': 265},\n",
       "  {'loss': 8.54723072052002, 'step': 266},\n",
       "  {'loss': 8.990455627441406, 'step': 267},\n",
       "  {'loss': 8.819659233093262, 'step': 268},\n",
       "  {'loss': 8.889856338500977, 'step': 269},\n",
       "  {'loss': 8.80689811706543, 'step': 270},\n",
       "  {'loss': 8.68995189666748, 'step': 271},\n",
       "  {'loss': 8.641124725341797, 'step': 272},\n",
       "  {'loss': 8.80321216583252, 'step': 273},\n",
       "  {'loss': 8.661818504333496, 'step': 274},\n",
       "  {'loss': 8.822019577026367, 'step': 275},\n",
       "  {'loss': 8.71877670288086, 'step': 276},\n",
       "  {'loss': 8.888238906860352, 'step': 277},\n",
       "  {'loss': 8.850825309753418, 'step': 278},\n",
       "  {'loss': 8.757631301879883, 'step': 279},\n",
       "  {'loss': 8.831743240356445, 'step': 280},\n",
       "  {'loss': 8.598512649536133, 'step': 281},\n",
       "  {'loss': 8.6563138961792, 'step': 282},\n",
       "  {'loss': 8.659075736999512, 'step': 283},\n",
       "  {'loss': 8.856963157653809, 'step': 284},\n",
       "  {'loss': 8.683056831359863, 'step': 285},\n",
       "  {'loss': 8.703252792358398, 'step': 286},\n",
       "  {'loss': 8.71760368347168, 'step': 287},\n",
       "  {'loss': 8.50401496887207, 'step': 288},\n",
       "  {'loss': 8.717348098754883, 'step': 289},\n",
       "  {'loss': 8.788076400756836, 'step': 290},\n",
       "  {'loss': 8.642265319824219, 'step': 291},\n",
       "  {'loss': 8.729260444641113, 'step': 292},\n",
       "  {'loss': 8.597262382507324, 'step': 293},\n",
       "  {'loss': 8.437006950378418, 'step': 294},\n",
       "  {'loss': 8.637210845947266, 'step': 295},\n",
       "  {'loss': 8.785807609558105, 'step': 296},\n",
       "  {'loss': 8.666900634765625, 'step': 297},\n",
       "  {'loss': 8.887128829956055, 'step': 298},\n",
       "  {'loss': 8.843318939208984, 'step': 299},\n",
       "  {'loss': 8.682428359985352, 'step': 300},\n",
       "  {'loss': 8.656373977661133, 'step': 301},\n",
       "  {'loss': 8.547762870788574, 'step': 302},\n",
       "  {'loss': 8.79814624786377, 'step': 303},\n",
       "  {'loss': 8.54286003112793, 'step': 304},\n",
       "  {'loss': 8.74468994140625, 'step': 305},\n",
       "  {'loss': 8.786928176879883, 'step': 306},\n",
       "  {'loss': 8.658790588378906, 'step': 307},\n",
       "  {'loss': 8.67253589630127, 'step': 308},\n",
       "  {'loss': 8.659226417541504, 'step': 309},\n",
       "  {'loss': 8.690098762512207, 'step': 310},\n",
       "  {'loss': 8.629892349243164, 'step': 311},\n",
       "  {'loss': 8.514575958251953, 'step': 312},\n",
       "  {'loss': 8.720386505126953, 'step': 313},\n",
       "  {'loss': 8.649149894714355, 'step': 314},\n",
       "  {'loss': 8.683838844299316, 'step': 315},\n",
       "  {'loss': 8.760747909545898, 'step': 316},\n",
       "  {'loss': 8.50149154663086, 'step': 317},\n",
       "  {'loss': 8.73776912689209, 'step': 318},\n",
       "  {'loss': 8.787997245788574, 'step': 319},\n",
       "  {'loss': 8.71859359741211, 'step': 320},\n",
       "  {'loss': 8.535606384277344, 'step': 321},\n",
       "  {'loss': 8.588956832885742, 'step': 322},\n",
       "  {'loss': 8.637584686279297, 'step': 323},\n",
       "  {'loss': 8.618596076965332, 'step': 324},\n",
       "  {'loss': 8.635401725769043, 'step': 325},\n",
       "  {'loss': 8.581416130065918, 'step': 326},\n",
       "  {'loss': 8.621212005615234, 'step': 327},\n",
       "  {'loss': 8.664250373840332, 'step': 328},\n",
       "  {'loss': 8.61622428894043, 'step': 329},\n",
       "  {'loss': 8.573439598083496, 'step': 330},\n",
       "  {'loss': 8.429200172424316, 'step': 331},\n",
       "  {'loss': 8.665876388549805, 'step': 332},\n",
       "  {'loss': 8.517274856567383, 'step': 333},\n",
       "  {'loss': 8.51968765258789, 'step': 334},\n",
       "  {'loss': 8.70827865600586, 'step': 335},\n",
       "  {'loss': 8.625079154968262, 'step': 336},\n",
       "  {'loss': 8.644512176513672, 'step': 337},\n",
       "  {'loss': 8.499407768249512, 'step': 338},\n",
       "  {'loss': 8.663261413574219, 'step': 339},\n",
       "  {'loss': 8.650629997253418, 'step': 340},\n",
       "  {'loss': 8.610466003417969, 'step': 341},\n",
       "  {'loss': 8.352932929992676, 'step': 342},\n",
       "  {'loss': 8.452920913696289, 'step': 343},\n",
       "  {'loss': 8.300736427307129, 'step': 344},\n",
       "  {'loss': 8.51846694946289, 'step': 345},\n",
       "  {'loss': 8.650123596191406, 'step': 346},\n",
       "  {'loss': 8.56052303314209, 'step': 347},\n",
       "  {'loss': 8.519867897033691, 'step': 348},\n",
       "  {'loss': 8.51663875579834, 'step': 349},\n",
       "  {'loss': 8.481672286987305, 'step': 350},\n",
       "  {'loss': 8.46656608581543, 'step': 351},\n",
       "  {'loss': 8.45077896118164, 'step': 352},\n",
       "  {'loss': 8.594993591308594, 'step': 353},\n",
       "  {'loss': 8.63070011138916, 'step': 354},\n",
       "  {'loss': 8.64052963256836, 'step': 355},\n",
       "  {'loss': 8.723430633544922, 'step': 356},\n",
       "  {'loss': 8.398032188415527, 'step': 357},\n",
       "  {'loss': 8.406326293945312, 'step': 358},\n",
       "  {'loss': 8.435787200927734, 'step': 359},\n",
       "  {'loss': 8.650368690490723, 'step': 360},\n",
       "  {'loss': 8.361128807067871, 'step': 361},\n",
       "  {'loss': 8.39635181427002, 'step': 362},\n",
       "  {'loss': 8.382181167602539, 'step': 363},\n",
       "  {'loss': 8.422292709350586, 'step': 364},\n",
       "  {'loss': 8.299302101135254, 'step': 365},\n",
       "  {'loss': 8.332137107849121, 'step': 366},\n",
       "  {'loss': 8.331846237182617, 'step': 367},\n",
       "  {'loss': 8.550846099853516, 'step': 368},\n",
       "  {'loss': 8.479788780212402, 'step': 369},\n",
       "  {'loss': 8.323003768920898, 'step': 370},\n",
       "  {'loss': 8.583909034729004, 'step': 371},\n",
       "  {'loss': 8.237923622131348, 'step': 372},\n",
       "  {'loss': 8.25442886352539, 'step': 373},\n",
       "  {'loss': 8.548093795776367, 'step': 374},\n",
       "  {'loss': 8.262472152709961, 'step': 375},\n",
       "  {'loss': 8.506821632385254, 'step': 376},\n",
       "  {'loss': 8.680696487426758, 'step': 377},\n",
       "  {'loss': 8.428462028503418, 'step': 378},\n",
       "  {'loss': 8.476308822631836, 'step': 379},\n",
       "  {'loss': 8.347206115722656, 'step': 380},\n",
       "  {'loss': 8.275992393493652, 'step': 381},\n",
       "  {'loss': 8.488994598388672, 'step': 382},\n",
       "  {'loss': 8.10950756072998, 'step': 383},\n",
       "  {'loss': 8.24510383605957, 'step': 384},\n",
       "  {'loss': 8.474987030029297, 'step': 385},\n",
       "  {'loss': 8.162385940551758, 'step': 386},\n",
       "  {'loss': 8.492452621459961, 'step': 387},\n",
       "  {'loss': 8.45319938659668, 'step': 388},\n",
       "  {'loss': 8.32604694366455, 'step': 389},\n",
       "  {'loss': 8.57326602935791, 'step': 390},\n",
       "  {'loss': 8.386674880981445, 'step': 391},\n",
       "  {'loss': 8.339153289794922, 'step': 392},\n",
       "  {'loss': 8.296455383300781, 'step': 393},\n",
       "  {'loss': 8.026095390319824, 'step': 394},\n",
       "  {'loss': 8.322244644165039, 'step': 395},\n",
       "  {'loss': 8.303525924682617, 'step': 396},\n",
       "  {'loss': 8.156100273132324, 'step': 397},\n",
       "  {'loss': 8.032267570495605, 'step': 398},\n",
       "  {'loss': 8.190345764160156, 'step': 399},\n",
       "  {'loss': 8.388103485107422, 'step': 400},\n",
       "  {'loss': 8.279502868652344, 'step': 401},\n",
       "  {'loss': 8.458852767944336, 'step': 402},\n",
       "  {'loss': 8.249776840209961, 'step': 403},\n",
       "  {'loss': 8.24227237701416, 'step': 404},\n",
       "  {'loss': 8.469518661499023, 'step': 405},\n",
       "  {'loss': 8.151318550109863, 'step': 406},\n",
       "  {'loss': 8.381696701049805, 'step': 407},\n",
       "  {'loss': 8.349331855773926, 'step': 408},\n",
       "  {'loss': 8.345230102539062, 'step': 409},\n",
       "  {'loss': 8.443326950073242, 'step': 410},\n",
       "  {'loss': 8.314427375793457, 'step': 411},\n",
       "  {'loss': 8.289935111999512, 'step': 412},\n",
       "  {'loss': 8.156458854675293, 'step': 413},\n",
       "  {'loss': 8.191102027893066, 'step': 414},\n",
       "  {'loss': 8.413723945617676, 'step': 415},\n",
       "  {'loss': 8.381078720092773, 'step': 416},\n",
       "  {'loss': 8.121923446655273, 'step': 417},\n",
       "  {'loss': 8.243027687072754, 'step': 418},\n",
       "  {'loss': 8.023468017578125, 'step': 419},\n",
       "  {'loss': 8.341687202453613, 'step': 420},\n",
       "  {'loss': 8.403969764709473, 'step': 421},\n",
       "  {'loss': 8.034883499145508, 'step': 422},\n",
       "  {'loss': 8.09855842590332, 'step': 423},\n",
       "  {'loss': 8.23582649230957, 'step': 424},\n",
       "  {'loss': 8.23161792755127, 'step': 425},\n",
       "  {'loss': 8.169029235839844, 'step': 426},\n",
       "  {'loss': 7.986759662628174, 'step': 427},\n",
       "  {'loss': 8.391189575195312, 'step': 428},\n",
       "  {'loss': 8.279129981994629, 'step': 429},\n",
       "  {'loss': 8.063817977905273, 'step': 430},\n",
       "  {'loss': 8.372984886169434, 'step': 431},\n",
       "  {'loss': 8.269021987915039, 'step': 432},\n",
       "  {'loss': 8.146788597106934, 'step': 433},\n",
       "  {'loss': 7.9376654624938965, 'step': 434},\n",
       "  {'loss': 8.151740074157715, 'step': 435},\n",
       "  {'loss': 8.063558578491211, 'step': 436},\n",
       "  {'loss': 7.987306118011475, 'step': 437},\n",
       "  {'loss': 8.068675994873047, 'step': 438},\n",
       "  {'loss': 8.228487014770508, 'step': 439},\n",
       "  {'loss': 8.099995613098145, 'step': 440},\n",
       "  {'loss': 8.217103958129883, 'step': 441},\n",
       "  {'loss': 8.160764694213867, 'step': 442},\n",
       "  {'loss': 8.315144538879395, 'step': 443},\n",
       "  {'loss': 8.196776390075684, 'step': 444},\n",
       "  {'loss': 8.192136764526367, 'step': 445},\n",
       "  {'loss': 8.256165504455566, 'step': 446},\n",
       "  {'loss': 8.341772079467773, 'step': 447},\n",
       "  {'loss': 7.887164115905762, 'step': 448},\n",
       "  {'loss': 8.05295181274414, 'step': 449},\n",
       "  {'loss': 8.172054290771484, 'step': 450},\n",
       "  {'loss': 8.163066864013672, 'step': 451},\n",
       "  {'loss': 7.934838771820068, 'step': 452},\n",
       "  {'loss': 8.1657133102417, 'step': 453},\n",
       "  {'loss': 8.067577362060547, 'step': 454},\n",
       "  {'loss': 7.782446384429932, 'step': 455},\n",
       "  {'loss': 8.057964324951172, 'step': 456},\n",
       "  {'loss': 7.98607873916626, 'step': 457},\n",
       "  {'loss': 8.245574951171875, 'step': 458},\n",
       "  {'loss': 7.963449954986572, 'step': 459},\n",
       "  {'loss': 8.175249099731445, 'step': 460},\n",
       "  {'loss': 8.19132137298584, 'step': 461},\n",
       "  {'loss': 8.11653995513916, 'step': 462},\n",
       "  {'loss': 7.88780403137207, 'step': 463},\n",
       "  {'loss': 7.880256175994873, 'step': 464},\n",
       "  {'loss': 7.945874214172363, 'step': 465},\n",
       "  {'loss': 7.961082458496094, 'step': 466},\n",
       "  {'loss': 8.01683235168457, 'step': 467},\n",
       "  {'loss': 7.752297401428223, 'step': 468},\n",
       "  {'loss': 7.98467493057251, 'step': 469},\n",
       "  {'loss': 7.8462018966674805, 'step': 470},\n",
       "  {'loss': 8.251471519470215, 'step': 471},\n",
       "  {'loss': 7.863866806030273, 'step': 472},\n",
       "  {'loss': 8.026667594909668, 'step': 473},\n",
       "  {'loss': 8.055121421813965, 'step': 474},\n",
       "  {'loss': 7.747836589813232, 'step': 475},\n",
       "  {'loss': 7.855240821838379, 'step': 476},\n",
       "  {'loss': 7.924991607666016, 'step': 477},\n",
       "  {'loss': 7.958405017852783, 'step': 478},\n",
       "  {'loss': 7.752425193786621, 'step': 479},\n",
       "  {'loss': 7.925333023071289, 'step': 480},\n",
       "  {'loss': 7.673107624053955, 'step': 481},\n",
       "  {'loss': 8.270515441894531, 'step': 482},\n",
       "  {'loss': 7.953527450561523, 'step': 483},\n",
       "  {'loss': 8.107476234436035, 'step': 484},\n",
       "  {'loss': 7.668939590454102, 'step': 485},\n",
       "  {'loss': 7.878506183624268, 'step': 486},\n",
       "  {'loss': 8.05875301361084, 'step': 487},\n",
       "  {'loss': 8.026162147521973, 'step': 488},\n",
       "  {'loss': 8.20295238494873, 'step': 489},\n",
       "  {'loss': 8.10754108428955, 'step': 490},\n",
       "  {'loss': 7.8981733322143555, 'step': 491},\n",
       "  {'loss': 7.754220485687256, 'step': 492},\n",
       "  {'loss': 8.123196601867676, 'step': 493},\n",
       "  {'loss': 7.9945902824401855, 'step': 494},\n",
       "  {'loss': 7.634234428405762, 'step': 495},\n",
       "  {'loss': 7.85506534576416, 'step': 496},\n",
       "  {'loss': 7.683794975280762, 'step': 497},\n",
       "  {'loss': 7.805235385894775, 'step': 498},\n",
       "  {'loss': 7.6742753982543945, 'step': 499},\n",
       "  {'loss': 7.802737712860107, 'step': 500},\n",
       "  {'loss': 7.8197808265686035, 'step': 501},\n",
       "  {'loss': 7.623452186584473, 'step': 502},\n",
       "  {'loss': 7.674893379211426, 'step': 503},\n",
       "  {'loss': 7.980107307434082, 'step': 504},\n",
       "  {'loss': 7.883018493652344, 'step': 505},\n",
       "  {'loss': 7.552188873291016, 'step': 506},\n",
       "  {'loss': 7.976924896240234, 'step': 507},\n",
       "  {'loss': 7.601591110229492, 'step': 508},\n",
       "  {'loss': 7.915145397186279, 'step': 509},\n",
       "  {'loss': 7.9531402587890625, 'step': 510},\n",
       "  {'loss': 7.884086608886719, 'step': 511},\n",
       "  {'loss': 7.832934379577637, 'step': 512},\n",
       "  {'loss': 7.623265743255615, 'step': 513},\n",
       "  {'loss': 7.745723247528076, 'step': 514},\n",
       "  {'loss': 7.654238700866699, 'step': 515},\n",
       "  {'loss': 7.808161735534668, 'step': 516},\n",
       "  {'loss': 7.967808723449707, 'step': 517},\n",
       "  {'loss': 7.834776878356934, 'step': 518},\n",
       "  {'loss': 7.952457904815674, 'step': 519},\n",
       "  {'loss': 7.84575891494751, 'step': 520},\n",
       "  {'loss': 7.5971784591674805, 'step': 521},\n",
       "  {'loss': 7.616504192352295, 'step': 522},\n",
       "  {'loss': 7.677937030792236, 'step': 523},\n",
       "  {'loss': 7.792447090148926, 'step': 524},\n",
       "  {'loss': 7.454806804656982, 'step': 525},\n",
       "  {'loss': 7.7406840324401855, 'step': 526},\n",
       "  {'loss': 7.71270751953125, 'step': 527},\n",
       "  {'loss': 7.697917461395264, 'step': 528},\n",
       "  {'loss': 7.698089122772217, 'step': 529},\n",
       "  {'loss': 7.38320779800415, 'step': 530},\n",
       "  {'loss': 7.763821125030518, 'step': 531},\n",
       "  {'loss': 7.5187225341796875, 'step': 532},\n",
       "  {'loss': 7.574438571929932, 'step': 533},\n",
       "  {'loss': 7.561459541320801, 'step': 534},\n",
       "  {'loss': 7.755963325500488, 'step': 535},\n",
       "  {'loss': 7.575839519500732, 'step': 536},\n",
       "  {'loss': 7.759202480316162, 'step': 537},\n",
       "  {'loss': 7.80471658706665, 'step': 538},\n",
       "  {'loss': 7.574631690979004, 'step': 539},\n",
       "  {'loss': 7.77430534362793, 'step': 540},\n",
       "  {'loss': 7.738511562347412, 'step': 541},\n",
       "  {'loss': 7.508213043212891, 'step': 542},\n",
       "  {'loss': 7.541123867034912, 'step': 543},\n",
       "  {'loss': 7.578123092651367, 'step': 544},\n",
       "  {'loss': 7.538346290588379, 'step': 545},\n",
       "  {'loss': 7.674654006958008, 'step': 546},\n",
       "  {'loss': 7.661852836608887, 'step': 547},\n",
       "  {'loss': 7.617372512817383, 'step': 548},\n",
       "  {'loss': 7.402965545654297, 'step': 549},\n",
       "  {'loss': 7.713715553283691, 'step': 550},\n",
       "  {'loss': 7.642049312591553, 'step': 551},\n",
       "  {'loss': 7.948065280914307, 'step': 552},\n",
       "  {'loss': 7.863307476043701, 'step': 553},\n",
       "  {'loss': 7.575323104858398, 'step': 554},\n",
       "  {'loss': 7.439660549163818, 'step': 555},\n",
       "  {'loss': 7.549602508544922, 'step': 556},\n",
       "  {'loss': 7.4573469161987305, 'step': 557},\n",
       "  {'loss': 7.37541389465332, 'step': 558},\n",
       "  {'loss': 7.404184818267822, 'step': 559},\n",
       "  {'loss': 7.538594722747803, 'step': 560},\n",
       "  {'loss': 7.570126056671143, 'step': 561},\n",
       "  {'loss': 7.442374229431152, 'step': 562},\n",
       "  {'loss': 7.528824806213379, 'step': 563},\n",
       "  {'loss': 7.689117908477783, 'step': 564},\n",
       "  {'loss': 7.54280424118042, 'step': 565},\n",
       "  {'loss': 7.764139175415039, 'step': 566},\n",
       "  {'loss': 7.391626358032227, 'step': 567},\n",
       "  {'loss': 7.343720436096191, 'step': 568},\n",
       "  {'loss': 7.534897327423096, 'step': 569},\n",
       "  {'loss': 7.478209495544434, 'step': 570},\n",
       "  {'loss': 7.337094306945801, 'step': 571},\n",
       "  {'loss': 7.263530731201172, 'step': 572},\n",
       "  {'loss': 7.107734680175781, 'step': 573},\n",
       "  {'loss': 7.728590488433838, 'step': 574},\n",
       "  {'loss': 7.543081283569336, 'step': 575},\n",
       "  {'loss': 7.501132488250732, 'step': 576},\n",
       "  {'loss': 7.0645551681518555, 'step': 577},\n",
       "  {'loss': 7.351832866668701, 'step': 578},\n",
       "  {'loss': 7.753512382507324, 'step': 579},\n",
       "  {'loss': 7.6314287185668945, 'step': 580},\n",
       "  {'loss': 7.334404468536377, 'step': 581},\n",
       "  {'loss': 7.456063747406006, 'step': 582},\n",
       "  {'loss': 7.2605791091918945, 'step': 583},\n",
       "  {'loss': 7.545652866363525, 'step': 584},\n",
       "  {'loss': 7.16420841217041, 'step': 585},\n",
       "  {'loss': 7.6003499031066895, 'step': 586},\n",
       "  {'loss': 7.347793102264404, 'step': 587},\n",
       "  {'loss': 7.499096393585205, 'step': 588},\n",
       "  {'loss': 7.5036163330078125, 'step': 589},\n",
       "  {'loss': 7.11732292175293, 'step': 590},\n",
       "  {'loss': 7.063774108886719, 'step': 591},\n",
       "  {'loss': 7.476491928100586, 'step': 592},\n",
       "  {'loss': 7.667438507080078, 'step': 593},\n",
       "  {'loss': 7.579757213592529, 'step': 594},\n",
       "  {'loss': 7.755599498748779, 'step': 595},\n",
       "  {'loss': 7.621546268463135, 'step': 596},\n",
       "  {'loss': 7.5712127685546875, 'step': 597},\n",
       "  {'loss': 7.201608180999756, 'step': 598},\n",
       "  {'loss': 7.575160980224609, 'step': 599},\n",
       "  {'loss': 7.299275875091553, 'step': 600},\n",
       "  {'loss': 7.474241256713867, 'step': 601},\n",
       "  {'loss': 7.355248928070068, 'step': 602},\n",
       "  {'loss': 7.3277974128723145, 'step': 603},\n",
       "  {'loss': 7.443240642547607, 'step': 604},\n",
       "  {'loss': 7.4748029708862305, 'step': 605},\n",
       "  {'loss': 7.281872272491455, 'step': 606},\n",
       "  {'loss': 7.2322916984558105, 'step': 607},\n",
       "  {'loss': 7.561732292175293, 'step': 608},\n",
       "  {'loss': 7.1530351638793945, 'step': 609},\n",
       "  {'loss': 7.169548988342285, 'step': 610},\n",
       "  {'loss': 6.871883392333984, 'step': 611},\n",
       "  {'loss': 7.670639991760254, 'step': 612},\n",
       "  {'loss': 7.176015853881836, 'step': 613},\n",
       "  {'loss': 7.334736347198486, 'step': 614},\n",
       "  {'loss': 7.063959121704102, 'step': 615},\n",
       "  {'loss': 7.4168314933776855, 'step': 616},\n",
       "  {'loss': 7.419042110443115, 'step': 617},\n",
       "  {'loss': 7.459080219268799, 'step': 618},\n",
       "  {'loss': 7.040624618530273, 'step': 619},\n",
       "  {'loss': 7.34957218170166, 'step': 620},\n",
       "  {'loss': 6.904488563537598, 'step': 621},\n",
       "  {'loss': 7.1392083168029785, 'step': 622},\n",
       "  {'loss': 7.379032611846924, 'step': 623},\n",
       "  {'loss': 7.227845191955566, 'step': 624},\n",
       "  {'loss': 7.323478698730469, 'step': 625},\n",
       "  {'loss': 7.382965087890625, 'step': 626},\n",
       "  {'loss': 7.269448757171631, 'step': 627},\n",
       "  {'loss': 7.233329772949219, 'step': 628},\n",
       "  {'loss': 7.018277645111084, 'step': 629},\n",
       "  {'loss': 7.125410556793213, 'step': 630},\n",
       "  {'loss': 7.3224711418151855, 'step': 631},\n",
       "  {'loss': 7.183382987976074, 'step': 632},\n",
       "  {'loss': 7.564347743988037, 'step': 633},\n",
       "  {'loss': 7.210575580596924, 'step': 634},\n",
       "  {'loss': 7.335386276245117, 'step': 635},\n",
       "  {'loss': 7.544443607330322, 'step': 636},\n",
       "  {'loss': 6.897097110748291, 'step': 637},\n",
       "  {'loss': 7.097145080566406, 'step': 638},\n",
       "  {'loss': 7.234475612640381, 'step': 639},\n",
       "  {'loss': 7.16341495513916, 'step': 640},\n",
       "  {'loss': 7.082449436187744, 'step': 641},\n",
       "  {'loss': 7.291557788848877, 'step': 642},\n",
       "  {'loss': 7.17813777923584, 'step': 643},\n",
       "  {'loss': 7.021999835968018, 'step': 644},\n",
       "  {'loss': 7.221175670623779, 'step': 645},\n",
       "  {'loss': 7.088931560516357, 'step': 646},\n",
       "  {'loss': 7.233983516693115, 'step': 647},\n",
       "  {'loss': 6.880917549133301, 'step': 648},\n",
       "  {'loss': 7.10094690322876, 'step': 649},\n",
       "  {'loss': 7.2618794441223145, 'step': 650},\n",
       "  {'loss': 7.070316314697266, 'step': 651},\n",
       "  {'loss': 6.727804183959961, 'step': 652},\n",
       "  {'loss': 7.200038909912109, 'step': 653},\n",
       "  {'loss': 7.07697057723999, 'step': 654},\n",
       "  {'loss': 6.779531002044678, 'step': 655},\n",
       "  {'loss': 7.235422611236572, 'step': 656},\n",
       "  {'loss': 7.414368629455566, 'step': 657},\n",
       "  {'loss': 7.137813091278076, 'step': 658},\n",
       "  {'loss': 7.2435173988342285, 'step': 659},\n",
       "  {'loss': 7.096873760223389, 'step': 660},\n",
       "  {'loss': 7.210554599761963, 'step': 661},\n",
       "  {'loss': 7.128813743591309, 'step': 662},\n",
       "  {'loss': 7.267638683319092, 'step': 663},\n",
       "  {'loss': 7.250959873199463, 'step': 664},\n",
       "  {'loss': 7.381091594696045, 'step': 665},\n",
       "  {'loss': 6.755768299102783, 'step': 666},\n",
       "  {'loss': 7.014893054962158, 'step': 667},\n",
       "  {'loss': 7.070833683013916, 'step': 668},\n",
       "  {'loss': 6.855494499206543, 'step': 669},\n",
       "  {'loss': 6.9647040367126465, 'step': 670},\n",
       "  {'loss': 7.350883960723877, 'step': 671},\n",
       "  {'loss': 6.581976413726807, 'step': 672},\n",
       "  {'loss': 7.27411413192749, 'step': 673},\n",
       "  {'loss': 6.948619365692139, 'step': 674},\n",
       "  {'loss': 7.083942413330078, 'step': 675},\n",
       "  {'loss': 7.072317600250244, 'step': 676},\n",
       "  {'loss': 7.167994499206543, 'step': 677},\n",
       "  {'loss': 6.959781169891357, 'step': 678},\n",
       "  {'loss': 7.053607940673828, 'step': 679},\n",
       "  {'loss': 6.936938285827637, 'step': 680},\n",
       "  {'loss': 7.0286478996276855, 'step': 681},\n",
       "  {'loss': 7.297170162200928, 'step': 682},\n",
       "  {'loss': 7.1734795570373535, 'step': 683},\n",
       "  {'loss': 6.857645034790039, 'step': 684},\n",
       "  {'loss': 7.205452919006348, 'step': 685},\n",
       "  {'loss': 7.157574653625488, 'step': 686},\n",
       "  {'loss': 6.899325847625732, 'step': 687},\n",
       "  {'loss': 6.811944484710693, 'step': 688},\n",
       "  {'loss': 7.061107158660889, 'step': 689},\n",
       "  {'loss': 6.671607971191406, 'step': 690},\n",
       "  {'loss': 6.865894794464111, 'step': 691},\n",
       "  {'loss': 6.928319454193115, 'step': 692},\n",
       "  {'loss': 7.278928279876709, 'step': 693},\n",
       "  {'loss': 6.638360023498535, 'step': 694},\n",
       "  {'loss': 7.0317230224609375, 'step': 695},\n",
       "  {'loss': 7.010473251342773, 'step': 696},\n",
       "  {'loss': 6.838586807250977, 'step': 697},\n",
       "  {'loss': 7.20792818069458, 'step': 698},\n",
       "  {'loss': 6.491509914398193, 'step': 699},\n",
       "  {'loss': 7.233540058135986, 'step': 700},\n",
       "  {'loss': 6.921062469482422, 'step': 701},\n",
       "  {'loss': 6.9504523277282715, 'step': 702},\n",
       "  {'loss': 6.945705413818359, 'step': 703},\n",
       "  {'loss': 6.65290641784668, 'step': 704},\n",
       "  {'loss': 6.606546878814697, 'step': 705},\n",
       "  {'loss': 7.113856315612793, 'step': 706},\n",
       "  {'loss': 6.834692001342773, 'step': 707},\n",
       "  {'loss': 6.9018778800964355, 'step': 708},\n",
       "  {'loss': 6.608719348907471, 'step': 709},\n",
       "  {'loss': 6.6852803230285645, 'step': 710},\n",
       "  {'loss': 6.409895896911621, 'step': 711},\n",
       "  {'loss': 6.8951616287231445, 'step': 712},\n",
       "  {'loss': 6.881501197814941, 'step': 713},\n",
       "  {'loss': 7.180202484130859, 'step': 714},\n",
       "  {'loss': 6.821073532104492, 'step': 715},\n",
       "  {'loss': 6.476235389709473, 'step': 716},\n",
       "  {'loss': 6.699018478393555, 'step': 717},\n",
       "  {'loss': 6.703683376312256, 'step': 718},\n",
       "  {'loss': 7.196874618530273, 'step': 719},\n",
       "  {'loss': 6.77877140045166, 'step': 720},\n",
       "  {'loss': 7.155152797698975, 'step': 721},\n",
       "  {'loss': 6.550099849700928, 'step': 722},\n",
       "  {'loss': 6.747347354888916, 'step': 723},\n",
       "  {'loss': 6.338167667388916, 'step': 724},\n",
       "  {'loss': 6.992344856262207, 'step': 725},\n",
       "  {'loss': 6.816182613372803, 'step': 726},\n",
       "  {'loss': 6.771712779998779, 'step': 727},\n",
       "  {'loss': 7.041903018951416, 'step': 728},\n",
       "  {'loss': 6.658247947692871, 'step': 729},\n",
       "  {'loss': 6.701721668243408, 'step': 730},\n",
       "  {'loss': 6.998523235321045, 'step': 731},\n",
       "  {'loss': 6.691590785980225, 'step': 732},\n",
       "  {'loss': 6.8059282302856445, 'step': 733},\n",
       "  {'loss': 7.148159503936768, 'step': 734},\n",
       "  {'loss': 6.708405017852783, 'step': 735},\n",
       "  {'loss': 6.640282154083252, 'step': 736},\n",
       "  {'loss': 6.853434085845947, 'step': 737},\n",
       "  {'loss': 6.654988765716553, 'step': 738},\n",
       "  {'loss': 6.358926296234131, 'step': 739},\n",
       "  {'loss': 6.730397701263428, 'step': 740},\n",
       "  {'loss': 6.797095775604248, 'step': 741},\n",
       "  {'loss': 6.75201940536499, 'step': 742},\n",
       "  {'loss': 6.475461483001709, 'step': 743},\n",
       "  {'loss': 7.030816078186035, 'step': 744},\n",
       "  {'loss': 6.858702659606934, 'step': 745},\n",
       "  {'loss': 6.619279861450195, 'step': 746},\n",
       "  {'loss': 6.242278099060059, 'step': 747},\n",
       "  {'loss': 6.274840831756592, 'step': 748},\n",
       "  {'loss': 6.676551342010498, 'step': 749},\n",
       "  {'loss': 6.492441654205322, 'step': 750},\n",
       "  {'loss': 6.98208475112915, 'step': 751},\n",
       "  {'loss': 6.461045265197754, 'step': 752},\n",
       "  {'loss': 6.423460483551025, 'step': 753},\n",
       "  {'loss': 6.3828229904174805, 'step': 754},\n",
       "  {'loss': 7.0380730628967285, 'step': 755},\n",
       "  {'loss': 6.800412178039551, 'step': 756},\n",
       "  {'loss': 6.771487712860107, 'step': 757},\n",
       "  {'loss': 7.051016330718994, 'step': 758},\n",
       "  {'loss': 6.6702561378479, 'step': 759},\n",
       "  {'loss': 6.271007537841797, 'step': 760},\n",
       "  {'loss': 6.212491989135742, 'step': 761},\n",
       "  {'loss': 6.679841995239258, 'step': 762},\n",
       "  {'loss': 6.666783809661865, 'step': 763},\n",
       "  {'loss': 6.653199195861816, 'step': 764},\n",
       "  {'loss': 6.837308406829834, 'step': 765},\n",
       "  {'loss': 6.486124038696289, 'step': 766},\n",
       "  {'loss': 6.964292526245117, 'step': 767},\n",
       "  {'loss': 6.869602203369141, 'step': 768},\n",
       "  {'loss': 6.7702741622924805, 'step': 769},\n",
       "  {'loss': 6.373586177825928, 'step': 770},\n",
       "  {'loss': 6.564524173736572, 'step': 771},\n",
       "  {'loss': 6.370936393737793, 'step': 772},\n",
       "  {'loss': 6.304485321044922, 'step': 773},\n",
       "  {'loss': 6.348942279815674, 'step': 774},\n",
       "  {'loss': 6.750998020172119, 'step': 775},\n",
       "  {'loss': 6.135613918304443, 'step': 776},\n",
       "  {'loss': 7.070189952850342, 'step': 777},\n",
       "  {'loss': 6.491796016693115, 'step': 778},\n",
       "  {'loss': 6.443453788757324, 'step': 779},\n",
       "  {'loss': 6.93723201751709, 'step': 780},\n",
       "  {'loss': 6.448551654815674, 'step': 781},\n",
       "  {'loss': 6.829986572265625, 'step': 782},\n",
       "  {'loss': 6.721899509429932, 'step': 783},\n",
       "  {'loss': 6.718744277954102, 'step': 784},\n",
       "  {'loss': 6.2706475257873535, 'step': 785},\n",
       "  {'loss': 6.4213104248046875, 'step': 786},\n",
       "  {'loss': 6.653752326965332, 'step': 787},\n",
       "  {'loss': 6.626402854919434, 'step': 788},\n",
       "  {'loss': 6.769937515258789, 'step': 789},\n",
       "  {'loss': 6.624207019805908, 'step': 790},\n",
       "  {'loss': 7.008984088897705, 'step': 791},\n",
       "  {'loss': 6.764901638031006, 'step': 792},\n",
       "  {'loss': 6.902143955230713, 'step': 793},\n",
       "  {'loss': 6.501473426818848, 'step': 794},\n",
       "  {'loss': 6.0991082191467285, 'step': 795},\n",
       "  {'loss': 6.396676063537598, 'step': 796},\n",
       "  {'loss': 6.6079840660095215, 'step': 797},\n",
       "  {'loss': 6.785189151763916, 'step': 798},\n",
       "  {'loss': 6.535006523132324, 'step': 799},\n",
       "  {'loss': 6.434908390045166, 'step': 800},\n",
       "  {'loss': 6.561550617218018, 'step': 801},\n",
       "  {'loss': 6.373966217041016, 'step': 802},\n",
       "  {'loss': 6.425358295440674, 'step': 803},\n",
       "  {'loss': 6.247760772705078, 'step': 804},\n",
       "  {'loss': 6.4308671951293945, 'step': 805},\n",
       "  {'loss': 6.607718467712402, 'step': 806},\n",
       "  {'loss': 6.768133163452148, 'step': 807},\n",
       "  {'loss': 6.549898624420166, 'step': 808},\n",
       "  {'loss': 6.516313552856445, 'step': 809},\n",
       "  {'loss': 6.613626003265381, 'step': 810},\n",
       "  {'loss': 6.450466632843018, 'step': 811},\n",
       "  {'loss': 6.767157077789307, 'step': 812},\n",
       "  {'loss': 6.474797248840332, 'step': 813},\n",
       "  {'loss': 6.698537349700928, 'step': 814},\n",
       "  {'loss': 6.795179843902588, 'step': 815},\n",
       "  {'loss': 6.334216117858887, 'step': 816},\n",
       "  {'loss': 6.852340221405029, 'step': 817},\n",
       "  {'loss': 6.108889102935791, 'step': 818},\n",
       "  {'loss': 6.1582932472229, 'step': 819},\n",
       "  {'loss': 6.383382797241211, 'step': 820},\n",
       "  {'loss': 6.130247592926025, 'step': 821},\n",
       "  {'loss': 6.454014301300049, 'step': 822},\n",
       "  {'loss': 6.357004165649414, 'step': 823},\n",
       "  {'loss': 6.270724773406982, 'step': 824},\n",
       "  {'loss': 6.1418962478637695, 'step': 825},\n",
       "  {'loss': 6.573232650756836, 'step': 826},\n",
       "  {'loss': 6.13216495513916, 'step': 827},\n",
       "  {'loss': 6.595534801483154, 'step': 828},\n",
       "  {'loss': 6.014074325561523, 'step': 829},\n",
       "  {'loss': 6.231898784637451, 'step': 830},\n",
       "  {'loss': 6.381300449371338, 'step': 831},\n",
       "  {'loss': 6.441524028778076, 'step': 832},\n",
       "  {'loss': 6.50509786605835, 'step': 833},\n",
       "  {'loss': 6.342873573303223, 'step': 834},\n",
       "  {'loss': 6.505883693695068, 'step': 835},\n",
       "  {'loss': 6.60174560546875, 'step': 836},\n",
       "  {'loss': 6.435044288635254, 'step': 837},\n",
       "  {'loss': 6.267453670501709, 'step': 838},\n",
       "  {'loss': 6.197514533996582, 'step': 839},\n",
       "  {'loss': 6.576354503631592, 'step': 840},\n",
       "  {'loss': 6.350658416748047, 'step': 841},\n",
       "  {'loss': 6.6106181144714355, 'step': 842},\n",
       "  {'loss': 6.2378644943237305, 'step': 843},\n",
       "  {'loss': 6.243220329284668, 'step': 844},\n",
       "  {'loss': 6.4279022216796875, 'step': 845},\n",
       "  {'loss': 6.48334264755249, 'step': 846},\n",
       "  {'loss': 6.241303443908691, 'step': 847},\n",
       "  {'loss': 6.057053089141846, 'step': 848},\n",
       "  {'loss': 6.095273017883301, 'step': 849},\n",
       "  {'loss': 6.138523578643799, 'step': 850},\n",
       "  {'loss': 6.247412204742432, 'step': 851},\n",
       "  {'loss': 6.082865238189697, 'step': 852},\n",
       "  {'loss': 6.058404445648193, 'step': 853},\n",
       "  {'loss': 6.6421942710876465, 'step': 854},\n",
       "  {'loss': 6.370232582092285, 'step': 855},\n",
       "  {'loss': 6.824501037597656, 'step': 856},\n",
       "  {'loss': 6.366448879241943, 'step': 857},\n",
       "  {'loss': 6.6022419929504395, 'step': 858},\n",
       "  {'loss': 6.690070629119873, 'step': 859},\n",
       "  {'loss': 5.94951868057251, 'step': 860},\n",
       "  {'loss': 6.412395477294922, 'step': 861},\n",
       "  {'loss': 6.546175956726074, 'step': 862},\n",
       "  {'loss': 6.723113536834717, 'step': 863},\n",
       "  {'loss': 6.617477893829346, 'step': 864},\n",
       "  {'loss': 6.161920070648193, 'step': 865},\n",
       "  {'loss': 6.731339454650879, 'step': 866},\n",
       "  {'loss': 5.905149936676025, 'step': 867},\n",
       "  {'loss': 6.242770671844482, 'step': 868},\n",
       "  {'loss': 6.761603832244873, 'step': 869},\n",
       "  {'loss': 6.549254894256592, 'step': 870},\n",
       "  {'loss': 6.081295013427734, 'step': 871},\n",
       "  {'loss': 5.792988300323486, 'step': 872},\n",
       "  {'loss': 5.952117443084717, 'step': 873},\n",
       "  {'loss': 6.0265116691589355, 'step': 874},\n",
       "  {'loss': 6.05133056640625, 'step': 875},\n",
       "  {'loss': 6.39686393737793, 'step': 876},\n",
       "  {'loss': 6.148279190063477, 'step': 877},\n",
       "  {'loss': 5.847620010375977, 'step': 878},\n",
       "  {'loss': 6.122174263000488, 'step': 879},\n",
       "  {'loss': 6.583841323852539, 'step': 880},\n",
       "  {'loss': 6.288015842437744, 'step': 881},\n",
       "  {'loss': 5.985568523406982, 'step': 882},\n",
       "  {'loss': 5.8980183601379395, 'step': 883},\n",
       "  {'loss': 6.355574607849121, 'step': 884},\n",
       "  {'loss': 6.38375186920166, 'step': 885},\n",
       "  {'loss': 6.550860404968262, 'step': 886},\n",
       "  {'loss': 6.178360462188721, 'step': 887},\n",
       "  {'loss': 6.192925453186035, 'step': 888},\n",
       "  {'loss': 6.495348930358887, 'step': 889},\n",
       "  {'loss': 6.566108226776123, 'step': 890},\n",
       "  {'loss': 6.157766342163086, 'step': 891},\n",
       "  {'loss': 6.366752624511719, 'step': 892},\n",
       "  {'loss': 6.568814754486084, 'step': 893},\n",
       "  {'loss': 6.0768561363220215, 'step': 894},\n",
       "  {'loss': 5.923153877258301, 'step': 895},\n",
       "  {'loss': 5.97401762008667, 'step': 896},\n",
       "  {'loss': 6.213046073913574, 'step': 897},\n",
       "  {'loss': 6.362200736999512, 'step': 898},\n",
       "  {'loss': 5.872248649597168, 'step': 899},\n",
       "  {'loss': 6.558959484100342, 'step': 900},\n",
       "  {'loss': 5.788500785827637, 'step': 901},\n",
       "  {'loss': 5.787539958953857, 'step': 902},\n",
       "  {'loss': 6.230277061462402, 'step': 903},\n",
       "  {'loss': 5.787574768066406, 'step': 904},\n",
       "  {'loss': 6.868687629699707, 'step': 905},\n",
       "  {'loss': 6.055898666381836, 'step': 906},\n",
       "  {'loss': 5.649701118469238, 'step': 907},\n",
       "  {'loss': 6.429406642913818, 'step': 908},\n",
       "  {'loss': 5.96832275390625, 'step': 909},\n",
       "  {'loss': 6.339225769042969, 'step': 910},\n",
       "  {'loss': 5.929661273956299, 'step': 911},\n",
       "  {'loss': 5.726000785827637, 'step': 912},\n",
       "  {'loss': 6.323074817657471, 'step': 913},\n",
       "  {'loss': 6.151116847991943, 'step': 914},\n",
       "  {'loss': 5.436618804931641, 'step': 915},\n",
       "  {'loss': 6.248456954956055, 'step': 916},\n",
       "  {'loss': 5.986039161682129, 'step': 917},\n",
       "  {'loss': 5.8570556640625, 'step': 918},\n",
       "  {'loss': 6.026274681091309, 'step': 919},\n",
       "  {'loss': 5.845066547393799, 'step': 920},\n",
       "  {'loss': 6.140651226043701, 'step': 921},\n",
       "  {'loss': 5.733924865722656, 'step': 922},\n",
       "  {'loss': 6.311446189880371, 'step': 923},\n",
       "  {'loss': 6.462681293487549, 'step': 924},\n",
       "  {'loss': 6.016454219818115, 'step': 925},\n",
       "  {'loss': 6.035895347595215, 'step': 926},\n",
       "  {'loss': 6.39762020111084, 'step': 927},\n",
       "  {'loss': 5.917646408081055, 'step': 928},\n",
       "  {'loss': 6.145869731903076, 'step': 929},\n",
       "  {'loss': 6.497987270355225, 'step': 930},\n",
       "  {'loss': 5.897854328155518, 'step': 931},\n",
       "  {'loss': 6.059278964996338, 'step': 932},\n",
       "  {'loss': 6.081830978393555, 'step': 933},\n",
       "  {'loss': 6.019891262054443, 'step': 934},\n",
       "  {'loss': 6.338781833648682, 'step': 935},\n",
       "  {'loss': 6.221980094909668, 'step': 936},\n",
       "  {'loss': 6.624582290649414, 'step': 937},\n",
       "  {'loss': 6.134873867034912, 'step': 938},\n",
       "  {'loss': 6.286379337310791, 'step': 939},\n",
       "  {'loss': 6.336732864379883, 'step': 940},\n",
       "  {'loss': 5.92794942855835, 'step': 941},\n",
       "  {'loss': 6.046794891357422, 'step': 942},\n",
       "  {'loss': 5.899456977844238, 'step': 943},\n",
       "  {'loss': 5.6347856521606445, 'step': 944},\n",
       "  {'loss': 6.358177185058594, 'step': 945},\n",
       "  {'loss': 6.499378681182861, 'step': 946},\n",
       "  {'loss': 5.735680103302002, 'step': 947},\n",
       "  {'loss': 6.092235565185547, 'step': 948},\n",
       "  {'loss': 6.014470100402832, 'step': 949},\n",
       "  {'loss': 6.097939968109131, 'step': 950},\n",
       "  {'loss': 6.57089376449585, 'step': 951},\n",
       "  {'loss': 5.928398132324219, 'step': 952},\n",
       "  {'loss': 6.375672817230225, 'step': 953},\n",
       "  {'loss': 6.1692705154418945, 'step': 954},\n",
       "  {'loss': 6.380002498626709, 'step': 955},\n",
       "  {'loss': 6.412543296813965, 'step': 956},\n",
       "  {'loss': 5.8729448318481445, 'step': 957},\n",
       "  {'loss': 6.319138526916504, 'step': 958},\n",
       "  {'loss': 5.997731685638428, 'step': 959},\n",
       "  {'loss': 6.215739727020264, 'step': 960},\n",
       "  {'loss': 5.831753253936768, 'step': 961},\n",
       "  {'loss': 6.253968238830566, 'step': 962},\n",
       "  {'loss': 5.796619892120361, 'step': 963},\n",
       "  {'loss': 6.353536128997803, 'step': 964},\n",
       "  {'loss': 5.657140254974365, 'step': 965},\n",
       "  {'loss': 6.382045745849609, 'step': 966},\n",
       "  {'loss': 6.0282721519470215, 'step': 967},\n",
       "  {'loss': 6.085155487060547, 'step': 968},\n",
       "  {'loss': 5.56928014755249, 'step': 969},\n",
       "  {'loss': 5.599862098693848, 'step': 970},\n",
       "  {'loss': 6.585090637207031, 'step': 971},\n",
       "  {'loss': 6.1276373863220215, 'step': 972},\n",
       "  {'loss': 6.320657730102539, 'step': 973},\n",
       "  {'loss': 5.850480079650879, 'step': 974},\n",
       "  {'loss': 6.157226085662842, 'step': 975},\n",
       "  {'loss': 5.950324535369873, 'step': 976},\n",
       "  {'loss': 6.241546154022217, 'step': 977},\n",
       "  {'loss': 6.045651912689209, 'step': 978},\n",
       "  {'loss': 6.104607582092285, 'step': 979},\n",
       "  {'loss': 5.781761646270752, 'step': 980},\n",
       "  {'loss': 6.256866931915283, 'step': 981},\n",
       "  {'loss': 5.853384971618652, 'step': 982},\n",
       "  {'loss': 6.344881534576416, 'step': 983},\n",
       "  {'loss': 5.94902229309082, 'step': 984},\n",
       "  {'loss': 5.754461765289307, 'step': 985},\n",
       "  {'loss': 6.304349422454834, 'step': 986},\n",
       "  {'loss': 5.971413612365723, 'step': 987},\n",
       "  {'loss': 5.774463653564453, 'step': 988},\n",
       "  {'loss': 6.214415550231934, 'step': 989},\n",
       "  {'loss': 6.674703598022461, 'step': 990},\n",
       "  {'loss': 5.917838096618652, 'step': 991},\n",
       "  {'loss': 5.492674827575684, 'step': 992},\n",
       "  {'loss': 6.056312084197998, 'step': 993},\n",
       "  {'loss': 6.047878742218018, 'step': 994},\n",
       "  {'loss': 5.807612419128418, 'step': 995},\n",
       "  {'loss': 5.944759368896484, 'step': 996},\n",
       "  {'loss': 6.298310279846191, 'step': 997},\n",
       "  {'loss': 5.882350444793701, 'step': 998},\n",
       "  {'loss': 5.809833526611328, 'step': 999},\n",
       "  {'loss': 6.431514739990234, 'step': 1000},\n",
       "  ...],\n",
       " 'val': [{'loss': 7.8073102831840515, 'step': 500},\n",
       "  {'loss': 5.971168911457061, 'step': 1000},\n",
       "  {'loss': 5.611986029148102, 'step': 1500},\n",
       "  {'loss': 5.328398966789246, 'step': 2000},\n",
       "  {'loss': 5.149145996570587, 'step': 2500},\n",
       "  {'loss': 4.920004880428314, 'step': 3000},\n",
       "  {'loss': 4.779552453756333, 'step': 3500},\n",
       "  {'loss': 4.681754398345947, 'step': 4000},\n",
       "  {'loss': 4.523450237512589, 'step': 4500},\n",
       "  {'loss': 4.423422479629517, 'step': 5000},\n",
       "  {'loss': 4.366112643480301, 'step': 5500},\n",
       "  {'loss': 4.294494533538819, 'step': 6000},\n",
       "  {'loss': 4.211083841323853, 'step': 6500},\n",
       "  {'loss': 4.165855121612549, 'step': 7000},\n",
       "  {'loss': 4.1183944880962375, 'step': 7500},\n",
       "  {'loss': 4.080932503938675, 'step': 8000},\n",
       "  {'loss': 4.0418670058250425, 'step': 8500},\n",
       "  {'loss': 4.0253000319004055, 'step': 9000},\n",
       "  {'loss': 3.978003990650177, 'step': 9500},\n",
       "  {'loss': 3.9496854186058044, 'step': 10000},\n",
       "  {'loss': 3.9268969297409058, 'step': 10500},\n",
       "  {'loss': 3.8950666069984434, 'step': 11000},\n",
       "  {'loss': 3.8760724484920503, 'step': 11500},\n",
       "  {'loss': 3.8436427235603334, 'step': 12000},\n",
       "  {'loss': 3.8339196741580963, 'step': 12500},\n",
       "  {'loss': 3.7939905762672423, 'step': 13000},\n",
       "  {'loss': 3.7756021201610563, 'step': 13500},\n",
       "  {'loss': 3.7568378925323485, 'step': 14000},\n",
       "  {'loss': 3.7428392708301543, 'step': 14500},\n",
       "  {'loss': 3.7278406739234926, 'step': 15000},\n",
       "  {'loss': 3.714115333557129, 'step': 15500},\n",
       "  {'loss': 3.6885502696037293, 'step': 16000},\n",
       "  {'loss': 3.673379212617874, 'step': 16500},\n",
       "  {'loss': 3.673127269744873, 'step': 17000},\n",
       "  {'loss': 3.6673579037189485, 'step': 17500},\n",
       "  {'loss': 3.657489222288132, 'step': 18000},\n",
       "  {'loss': 3.6370930850505827, 'step': 18500},\n",
       "  {'loss': 3.615221434831619, 'step': 19000},\n",
       "  {'loss': 3.62474205493927, 'step': 19500},\n",
       "  {'loss': 3.6018655478954313, 'step': 20000},\n",
       "  {'loss': 3.6036490976810454, 'step': 20500},\n",
       "  {'loss': 3.5887611150741576, 'step': 21000},\n",
       "  {'loss': 3.584767359495163, 'step': 21500},\n",
       "  {'loss': 3.585175263881683, 'step': 22000},\n",
       "  {'loss': 3.5711297392845154, 'step': 22500},\n",
       "  {'loss': 3.553194147348404, 'step': 23000},\n",
       "  {'loss': 3.546617865562439, 'step': 23500},\n",
       "  {'loss': 3.554442662000656, 'step': 24000},\n",
       "  {'loss': 3.5421711027622225, 'step': 24500},\n",
       "  {'loss': 3.5452450692653654, 'step': 25000},\n",
       "  {'loss': 3.532296723127365, 'step': 25500},\n",
       "  {'loss': 3.5292220175266267, 'step': 26000},\n",
       "  {'loss': 3.5292272686958315, 'step': 26500},\n",
       "  {'loss': 3.5220127582550047, 'step': 27000},\n",
       "  {'loss': 3.519837313890457, 'step': 27500},\n",
       "  {'loss': 3.5219656825065613, 'step': 28000},\n",
       "  {'loss': 3.513019025325775, 'step': 28500},\n",
       "  {'loss': 3.508002746105194, 'step': 29000},\n",
       "  {'loss': 3.5013299167156218, 'step': 29500},\n",
       "  {'loss': 3.5098833620548247, 'step': 30000},\n",
       "  {'loss': 3.4986371040344237, 'step': 30500},\n",
       "  {'loss': 3.5034032821655274, 'step': 31000},\n",
       "  {'loss': 3.5008905708789824, 'step': 31500},\n",
       "  {'loss': 3.4970364212989806, 'step': 32000},\n",
       "  {'loss': 3.496157306432724, 'step': 32500},\n",
       "  {'loss': 3.499789595603943, 'step': 33000},\n",
       "  {'loss': 3.489778792858124, 'step': 33500},\n",
       "  {'loss': 3.494791680574417, 'step': 34000},\n",
       "  {'loss': 3.493950551748276, 'step': 34500},\n",
       "  {'loss': 3.493501269817352, 'step': 35000},\n",
       "  {'loss': 3.4913229942321777, 'step': 35500},\n",
       "  {'loss': 3.498767763376236, 'step': 36000},\n",
       "  {'loss': 3.4981051087379456, 'step': 36500},\n",
       "  {'loss': 3.4980940401554106, 'step': 37000},\n",
       "  {'loss': 3.492477059364319, 'step': 37500},\n",
       "  {'loss': 3.50477329492569, 'step': 38000},\n",
       "  {'loss': 3.4910234332084658, 'step': 38500}]}"
      ]
     },
     "execution_count": 383,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 383
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# 推理",
   "id": "6a61add6c17be6c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-25T03:22:01.033071Z",
     "start_time": "2025-03-25T03:22:01.015659Z"
    }
   },
   "cell_type": "code",
   "source": "exp_name",
   "id": "28e288f9fa662c14",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'translate-transformer-not-share'"
      ]
     },
     "execution_count": 384,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 384
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-25T03:22:13.953795Z",
     "start_time": "2025-03-25T03:22:13.795781Z"
    }
   },
   "cell_type": "code",
   "source": "!ls checkpoints/translate-transformer-not-share -l",
   "id": "481191a753366580",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 281580\n",
      "-rw-r--r-- 1 35493 197609 288336196 Mar 24 20:31 best.ckpt\n"
     ]
    }
   ],
   "execution_count": 385
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-25T03:22:41.560695Z",
     "start_time": "2025-03-25T03:22:41.147712Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "\n",
    "state_dict = torch.load(f\"checkpoints/translate-transformer-not-share/best.ckpt\", map_location=device)\n",
    "\n",
    "# state_dict1 = torch.load(\"epoch125-step132426.ckpt\", map_location=\"cpu\")\n",
    "# state_dict = state_dict1[\"state_dict\"]\n",
    "\n",
    "# update keys by dropping `model`\n",
    "# for key in list(state_dict):\n",
    "#     state_dict[key.replace(\"model.\", \"\")] = state_dict.pop(key)\n"
   ],
   "id": "91bc7d2da2938b95",
   "outputs": [],
   "execution_count": 386
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-25T03:22:57.980853Z",
     "start_time": "2025-03-25T03:22:57.871763Z"
    }
   },
   "cell_type": "code",
   "source": "!rm -r wmt16/.cache",
   "id": "ea0d0d937f5e2920",
   "outputs": [],
   "execution_count": 387
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-25T03:23:49.928112Z",
     "start_time": "2025-03-25T03:23:30.242654Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "\n",
    "# load checkpoints\n",
    "model = TransformerModel(config)\n",
    "model.load_state_dict(state_dict)\n",
    "\n",
    "loss_fct = CrossEntropyWithPadding(config)\n",
    "# from dataset import LangPairDataset\n",
    "test_ds = LangPairDataset(\"test\", max_length=128, data_dir=\"./wmt16\")\n",
    "test_dl = DataLoader(test_ds, batch_size=1, collate_fn=partial(collate_fct, tokenizer=tokenizer))\n",
    "\n",
    "model = model.to(device)\n",
    "model.eval()\n",
    "collect = {}\n",
    "loss_collect = []\n",
    "\n",
    "predictions = []\n",
    "answers = []\n",
    "# 初始化BLEU分数列表\n",
    "bleu_scores = []\n",
    "for idx, batch in tqdm(enumerate(test_dl)):\n",
    "    encoder_inputs = batch[\"encoder_inputs\"]\n",
    "    encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "    decoder_inputs = batch[\"decoder_inputs\"]\n",
    "    decoder_labels = batch[\"decoder_labels\"]\n",
    "    # print(decoder_labels.cpu())\n",
    "    # decoder_labels1=tokenizer.decode(decoder_labels.cpu().numpy())\n",
    "    # print(decoder_labels1)\n",
    "    # 前向计算\n",
    "    outputs = model(\n",
    "        encoder_inputs=encoder_inputs,\n",
    "        decoder_inputs=decoder_inputs,\n",
    "        encoder_inputs_mask=encoder_inputs_mask\n",
    "    )\n",
    "    loss = loss_fct(outputs.logits, decoder_labels)  # 验证集损失\n",
    "\n",
    "    # print(outputs.logits.shape, decoder_labels.shape)\n",
    "\n",
    "    # loss = loss_fct(outputs.logits[:, :decoder_labels.shape[1]], decoder_labels)         # 验证集损失\n",
    "    # outputs = model.infer(encoder_inputs=encoder_inputs)\n",
    "    # print(outputs.logits.shape)\n",
    "    preds = outputs.logits.argmax(dim=-1)  # 预测结果，[1,seq_len]\n",
    "    # print(preds.shape)\n",
    "    #把preds转为英文单词\n",
    "    preds = tokenizer.decode(preds.cpu().numpy())  #['预测句子']\n",
    "    # predictions.append(preds)\n",
    "    # print(preds)\n",
    "    #把decoder_labels转为英文单词\n",
    "    decoder_labels = tokenizer.decode(decoder_labels.cpu().numpy())  #['标签句子']\n",
    "    # answers.append(decoder_labels)\n",
    "    # print(decoder_labels)\n",
    "    belu = sentence_bleu([decoder_labels[0].split()], preds[0].split(), weights=(1, 0, 0, 0))\n",
    "    bleu_scores.append(belu)\n",
    "    collect[idx] = {\"loss\": loss.item(), \"src_inputs\": encoder_inputs, \"trg_inputs\": decoder_inputs,\n",
    "                    \"mask\": encoder_inputs_mask, \"trg_labels\": decoder_labels, \"preds\": preds}\n",
    "    loss_collect.append(loss.item())\n",
    "    # break\n",
    "\n",
    "# sort collect by value\n",
    "collect = sorted(collect.items(), key=lambda x: x[1][\"loss\"])\n",
    "print(f\"testing loss: {np.array(loss_collect).mean()}\")\n",
    "sum(bleu_scores) / len(bleu_scores)"
   ],
   "id": "36436a7231764d39",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save cache to wmt16\\.cache\\de2en_test_128.npy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]C:\\Users\\35493\\AppData\\Roaming\\Python\\Python312\\site-packages\\nltk\\translate\\bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "11it [00:00, 36.14it/s]C:\\Users\\35493\\AppData\\Roaming\\Python\\Python312\\site-packages\\nltk\\translate\\bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 3-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "26it [00:00, 42.87it/s]C:\\Users\\35493\\AppData\\Roaming\\Python\\Python312\\site-packages\\nltk\\translate\\bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 2-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "966it [00:17, 54.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "testing loss: 3.469803318350458\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5660292763261168"
      ]
     },
     "execution_count": 388,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 388
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-25T03:26:44.283611Z",
     "start_time": "2025-03-25T03:26:39.052757Z"
    }
   },
   "cell_type": "code",
   "source": [
    "!pip install Cython\n",
    "!pip install fastBPE"
   ],
   "id": "112e1a7cb0a07102",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: Cython in c:\\users\\35493\\appdata\\roaming\\python\\python312\\site-packages (3.0.12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[notice] A new release of pip is available: 24.0 -> 25.0.1\n",
      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting fastBPE\n",
      "  Using cached fastBPE-0.1.0.tar.gz (35 kB)\n",
      "  Preparing metadata (setup.py): started\n",
      "  Preparing metadata (setup.py): finished with status 'done'\n",
      "Building wheels for collected packages: fastBPE\n",
      "  Building wheel for fastBPE (setup.py): started\n",
      "  Building wheel for fastBPE (setup.py): finished with status 'error'\n",
      "  Running setup.py clean for fastBPE\n",
      "Failed to build fastBPE\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  error: subprocess-exited-with-error\n",
      "  \n",
      "  python setup.py bdist_wheel did not run successfully.\n",
      "  exit code: 1\n",
      "  \n",
      "  [6 lines of output]\n",
      "  running bdist_wheel\n",
      "  running build\n",
      "  running build_py\n",
      "  running build_ext\n",
      "  building 'fastBPE' extension\n",
      "  error: Microsoft Visual C++ 14.0 or greater is required. Get it with \"Microsoft C++ Build Tools\": https://visualstudio.microsoft.com/visual-cpp-build-tools/\n",
      "  [end of output]\n",
      "  \n",
      "  note: This error originates from a subprocess, and is likely not a problem with pip.\n",
      "  ERROR: Failed building wheel for fastBPE\n",
      "ERROR: Could not build wheels for fastBPE, which is required to install pyproject.toml-based projects\n",
      "\n",
      "[notice] A new release of pip is available: 24.0 -> 25.0.1\n",
      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
     ]
    }
   ],
   "execution_count": 392
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-25T03:23:55.582743Z",
     "start_time": "2025-03-25T03:23:55.416698Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import re\n",
    "from fastBPE import fastBPE\n",
    "from sacremoses import MosesDetokenizer, MosesTokenizer\n",
    "\n",
    "\n",
    "# `MosesTokenizer` 和 `MosesDetokenizer` 是来自 `sacremoses` 库的工具，用于自然语言处理中的分词（Tokenization）和去标记化（Detokenization）。这些工具主要用于对文本进行预处理和后处理，通常在处理自然语言处理任务时会用到。\n",
    "#\n",
    "# ### MosesTokenizer：\n",
    "# - **作用**：将原始文本分割成单词和标点符号。\n",
    "# - **特点**：基于 Moses 翻译工具中使用的分词方法。\n",
    "# - **功能**：\n",
    "#   - 将句子分割成单词和标点符号。\n",
    "#   - 处理缩写、连字符、标点等特殊情况。\n",
    "#   - 对文本进行标记化，方便后续处理。\n",
    "#\n",
    "# ### MosesDetokenizer：\n",
    "# - **作用**：将分词后的文本重新组合成原始的句子。\n",
    "# - **特点**：用于对分词后的文本进行还原，使其恢复为可读的句子形式。\n",
    "# - **功能**：\n",
    "#   - 将分词后的单词和标点符号重新组合成句子。\n",
    "#   - 处理分词后的标点、缩写等情况，使得结果更加自然和可读。\n",
    "#\n",
    "# 这些工具通常在文本预处理和后处理过程中使用，对输入的文本进行标记化和去标记化，是一种常用的处理方式。在自然语言处理任务中，对文本进行正确的分词和还原是很重要的，而 `MosesTokenizer` 和 `MosesDetokenizer` 提供了方便、高效的工具来处理这些任务。\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, model, src_tokenizer, trg_tokenizer):\n",
    "        self.bpe = fastBPE(\"./wmt16/bpe.10000\", \"./wmt16/vocab\")\n",
    "        self.mose_tokenizer = MosesTokenizer(lang=\"de\")\n",
    "        self.mose_detokenizer = MosesDetokenizer(lang=\"en\")\n",
    "        self.model = model\n",
    "        self.model.eval()\n",
    "        self.src_tokenizer = src_tokenizer\n",
    "        self.trg_tokenizer = trg_tokenizer\n",
    "        self.pattern = re.compile(r'(@@ )|(@@ ?$)')\n",
    "\n",
    "    def draw_attention_map(self, attn_scores, cross_attn_scores, src_words_list, trg_words_list):\n",
    "        \"\"\"绘制注意力热力图\n",
    "        attn_scores (numpy.ndarray): 表示自注意力机制（self-attention）分数。\n",
    "        cross_attn_scores (numpy.ndarray): 表示交叉注意力机制的注意力分数。\n",
    "        src_words_list (list): 源语言句子的单词列表。\n",
    "        trg_words_list (list): 目标语言句子的单词列表。\n",
    "        \"\"\"\n",
    "        assert len(attn_scores.shape) == 3, \"attn_scores shape should be \" \\\n",
    "                                            f\"[num heads, target sequence length, target sequence length], but got {attn_scores.shape}\"\n",
    "        attn_scores = attn_scores[:, :len(trg_words_list), :len(trg_words_list)]\n",
    "\n",
    "        assert len(cross_attn_scores.shape) == 3, \"attn_scores shape should be \" \\\n",
    "                                                  f\"[num heads, target sequence length, source sequence length], but got {cross_attn_scores.shape}\"\n",
    "        cross_attn_scores = cross_attn_scores[:, :len(trg_words_list), :len(src_words_list)]\n",
    "\n",
    "        num_heads, trg_len, src_len = cross_attn_scores.shape\n",
    "\n",
    "        fig = plt.figure(figsize=(10, 5), constrained_layout=True)  # constrained_layout=True 自动调整子图参数，使之填充整个图像区域\n",
    "        grid = plt.GridSpec(trg_len, trg_len + src_len, wspace=0.1, hspace=0.1)  # wspace,hspace 控制子图之间的间距\n",
    "        #下面是attn_scores的热力图\n",
    "        self_map = fig.add_subplot(grid[:, :trg_len])  #  添加子图\n",
    "        self_map.matshow(attn_scores.mean(dim=0), cmap='viridis')  # 绘制热力图，cmap表示颜色,dim=0表示对第0维求均值\n",
    "        self_map.set_yticks(range(trg_len), trg_words_list, fontsize=10)\n",
    "        self_map.set_xticks(range(trg_len), [\"[BOS]\"] + trg_words_list[:-1], rotation=90)\n",
    "        #下面是cross_attn_scores的热力图\n",
    "        cross_map = fig.add_subplot(grid[:, trg_len:])\n",
    "        cross_map.matshow(cross_attn_scores.mean(dim=0), cmap='viridis')\n",
    "        cross_map.set_yticks(range(trg_len), [], fontsize=6)\n",
    "        cross_map.set_xticks(range(src_len), src_words_list, rotation=90)\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "    def draw_attention_maps(self, attn_scores, cross_attn_scores, src_words_list, trg_words_list, heads_list):\n",
    "        \"\"\"绘制注意力热力图\n",
    "\n",
    "        Args:\n",
    "            - scores (numpy.ndarray): shape = [source sequence length, target sequence length]\n",
    "        \"\"\"\n",
    "        assert len(attn_scores.shape) == 3, \"attn_scores shape should be \" \\\n",
    "                                            f\"[num heads, target sequence length, target sequence length], but got {attn_scores.shape}\"\n",
    "        attn_scores = attn_scores[:, :len(trg_words_list), :len(trg_words_list)]\n",
    "\n",
    "        assert len(cross_attn_scores.shape) == 3, \"attn_scores shape should be \" \\\n",
    "                                                  f\"[num heads, target sequence length, source sequence length], but got {cross_attn_scores.shape}\"\n",
    "        cross_attn_scores = cross_attn_scores[:, :len(trg_words_list), :len(src_words_list)]\n",
    "        # cross_attn_scores = cross_attn_scores[:, :len(src_words_list), :len(src_words_list)]\n",
    "\n",
    "        num_heads, trg_len, src_len = cross_attn_scores.shape\n",
    "        fig, axes = plt.subplots(2, len(heads_list), figsize=(5 * len(heads_list), 10))\n",
    "        for i, heads_idx in enumerate(heads_list):\n",
    "            axes[0, i].matshow(attn_scores[heads_idx], cmap='viridis')\n",
    "            axes[0, i].set_yticks(range(trg_len), trg_words_list)\n",
    "            axes[0, i].set_xticks(range(trg_len), [\"[BOS]\"] + trg_words_list[:-1], rotation=90)\n",
    "            axes[0, i].set_title(f\"head {heads_idx}\")\n",
    "            axes[1, i].matshow(cross_attn_scores[heads_idx], cmap='viridis')\n",
    "            axes[1, i].set_yticks(range(trg_len), trg_words_list)\n",
    "            axes[1, i].set_xticks(range(src_len), src_words_list, rotation=90)\n",
    "            axes[1, i].set_title(f\"head {heads_idx}\")\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "    def __call__(self, sentence_list, heads_list=None, layer_idx=-1):\n",
    "        # 将输入句子列表转换为小写，并使用 MosesTokenizer 进行分词处理。\n",
    "        sentence_list = [\" \".join(self.mose_tokenizer.tokenize(s.lower())) for s in sentence_list]\n",
    "        # 将分词后的结果进行 BPE 编码，得到 tokens_list。\n",
    "        tokens_list = [s.split() for s in self.bpe.apply(sentence_list)]\n",
    "        # 使用 src_tokenizer 对 tokens_list 进行编码，同时添加起始标记 ([BOS]) 和结束标记 ([EOS])。\n",
    "        encoder_input, attn_mask = self.src_tokenizer.encode(\n",
    "            tokens_list,\n",
    "            add_bos=True,\n",
    "            add_eos=True,\n",
    "            return_mask=True,\n",
    "        )\n",
    "        encoder_input = torch.Tensor(encoder_input).to(dtype=torch.int64)\n",
    "        # 使用模型的 infer 方法对编码器输入进行推理，得到输出结果 outputs\n",
    "        outputs = model.infer(encoder_inputs=encoder_input, encoder_inputs_mask=attn_mask)\n",
    "\n",
    "        preds = outputs.preds.numpy()\n",
    "        # 使用目标语言的 trg_tokenizer 对预测序列进行解码，得到解码后的目标语言句子列表 trg_decoded。\n",
    "        trg_decoded = self.trg_tokenizer.decode(preds, split=True, remove_eos=False, remove_bos=False, remove_pad=False)\n",
    "        # 使用源语言的 src_tokenizer 对编码器输入进行解码，得到解码后的源语言句子列表 src_decoded。为下面绘制热力图做准备。\n",
    "        src_decoded = self.src_tokenizer.decode(\n",
    "            encoder_input.numpy(),\n",
    "            split=True,\n",
    "            remove_bos=False,\n",
    "            remove_eos=False\n",
    "        )\n",
    "\n",
    "        # post processed attn scores\n",
    "        # outputs.decoder_attentions[-1]  # the last layer of self-attention scores\n",
    "\n",
    "        # draw the attention map of the last decoder block\n",
    "        for attn_score, cross_attn_score, src, trg in zip(\n",
    "                outputs.decoder_self_attn_scores[layer_idx], outputs.decoder_cross_attn_scores[layer_idx], src_decoded,\n",
    "                trg_decoded):\n",
    "            if heads_list is None:  # 如果没有指定heads_list，就画单个热力图\n",
    "                self.draw_attention_map(\n",
    "                    attn_score,\n",
    "                    cross_attn_score,\n",
    "                    src,\n",
    "                    trg,\n",
    "                )\n",
    "            else:  # 如果指定了heads_list，就画多个热力图\n",
    "                self.draw_attention_maps(\n",
    "                    attn_score,\n",
    "                    cross_attn_score,\n",
    "                    src,\n",
    "                    trg,\n",
    "                    heads_list=heads_list,\n",
    "                )\n",
    "        return [self.mose_detokenizer.tokenize(self.pattern.sub(\"\", s).split()) for s in\n",
    "                self.trg_tokenizer.decode(preds)]  #将解码后的目标语言句子列表返回，并使用 mose_detokenizer 进行去标记化，最终得到翻译后的结果。\n",
    "\n",
    "\n",
    "# sentence_list = [\n",
    "#     \"Mann in einem kleinen weißen Boot auf einem See.\",  # Man in a small white boat on a lake.\n",
    "#     \"Ein Mann mit einem Eimer und ein Mädchen mit einem Hut am Strand.\", # A man with a bucket and a girl in a hat on the beach.\n",
    "#     \"Drei Männer auf Pferden während eines Rennens.\",  # Three men on horses during a race.\n",
    "#     \"Ein Mann und eine Frau essen zu Abend\",  # 一个男人和一个女人在吃晚餐\n",
    "# ]\n",
    "sentence_list = [\n",
    "    \"Mann in einem kleinen weißen Boot auf einem See.\",  # Man in a small white boat on a lake.\n",
    "    \"Ein Mann mit einem Eimer und ein Mädchen mit einem Hut am Strand.\",\n",
    "    # A man with a bucket and a girl in a hat on the beach.\n",
    "    \"Drei Männer auf Pferden während eines Rennens.\",  # Three men on horses during a race.\n",
    "    \"Ein Mann und eine Frau essen zu Abend\",  # A man and a woman eating dinner\n",
    "]\n",
    "\n",
    "# load checkpoints\n",
    "model = TransformerModel(config)\n",
    "model.load_state_dict(state_dict)\n",
    "translator = Translator(model.cpu(), tokenizer, tokenizer)\n",
    "translator(\n",
    "    sentence_list,\n",
    "    layer_idx=-1,\n",
    "    # heads_list=[0, 1, 2, 3, 4, 5, 6, 7]\n",
    ")\n"
   ],
   "id": "52c1b2bbd24791ea",
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'fastBPE'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mModuleNotFoundError\u001B[0m                       Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[389], line 2\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mre\u001B[39;00m\n\u001B[1;32m----> 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mfastBPE\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m fastBPE\n\u001B[0;32m      3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01msacremoses\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m MosesDetokenizer, MosesTokenizer\n\u001B[0;32m      5\u001B[0m \u001B[38;5;66;03m# `MosesTokenizer` 和 `MosesDetokenizer` 是来自 `sacremoses` 库的工具，用于自然语言处理中的分词（Tokenization）和去标记化（Detokenization）。这些工具主要用于对文本进行预处理和后处理，通常在处理自然语言处理任务时会用到。\u001B[39;00m\n\u001B[0;32m      6\u001B[0m \u001B[38;5;66;03m#\u001B[39;00m\n\u001B[0;32m      7\u001B[0m \u001B[38;5;66;03m# ### MosesTokenizer：\u001B[39;00m\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m     21\u001B[0m \u001B[38;5;66;03m#\u001B[39;00m\n\u001B[0;32m     22\u001B[0m \u001B[38;5;66;03m# 这些工具通常在文本预处理和后处理过程中使用，对输入的文本进行标记化和去标记化，是一种常用的处理方式。在自然语言处理任务中，对文本进行正确的分词和还原是很重要的，而 `MosesTokenizer` 和 `MosesDetokenizer` 提供了方便、高效的工具来处理这些任务。\u001B[39;00m\n",
      "\u001B[1;31mModuleNotFoundError\u001B[0m: No module named 'fastBPE'"
     ]
    }
   ],
   "execution_count": 389
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "253381e840295509"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
